【原创】OpenStack Swift源码分析（一）builder文件的生成

最近开始Swift源码分析，无非就是不想纸上谈兵，到底是用什么算法，和方法实现的功能，因此决定开始写源码分析。

    当你安装完swift,你须要作的第一件事情就是建立ring文件，而你用的第一个命令就是swift-ring-builder。swift-ring-builder文件位于源码的bin目录下，是swift最基本的命令，它与swift/common/ring/下的文件一块儿实现ring文件建立，添加，平衡，等操做。

    对于初学者（好比说我）重写源码片断，能够更加深刻的了解源码的原理，同时还能对python语言以及相关的库有更深的了解，swift-ring-builder中主要的功能实现就是在Commands类中，好比default(),create(),add(),rebalance()等， 而后main方法会根据你提供的相应参数，来提供执行相应的方法，而后其中的方法会调用/swift/common/ring/下的builder.py中相应的方法最终实现相应的操做，

当咱们经过create建立account.builder文件的时候，commod == argv[2] 也就是create 而后执行create来建立account.builder。以后的操做，只要是存在account.builder文件，就会打开这个文件，生产builder实例，来进行相应的操做。其中的 default方法是显示当前的builder信息，能够用来在rebalance以前 检查add的device 。

其中reblance是最重要的方法，当中会涉及到/swift/common/ring下的ring.py builder.py utils.py文件，涉及到了一致性哈希算法和策略的实现，下个博客会具体分析。

代码片断：基本就是把源码抽出来，没写什么注释和异常处理，生成的builder文件，能够经过diff命令，比较是否跟使用swift-ring-builder命令建立的builder文件同样（答案固然是确定的）。

 

#! /usr/bin/env python

from sys import argv, exit, modules
import cPickle as pickle
from os.path import basename, dirname, exists, join as pathjoin
from itertools import islice, izip

from builder import RingBuilder


MAJOR_VERSION = 1
MINOR_VERSION = 3
EXIT_SUCCESS = 0
EXIT_WARNING = 1
EXIT_ERROR = 2


class Commands:

    def unknown():
        print 'Unknown command: %s' % argv[2]
        exit(EXIT_ERROR)
    
    def create():
        """
        my-ring-builder <builder_file> create <part_power> <replicas>
                                                <min_part_hours>
        """

        if len(argv) < 6:
            print Commands.create.__doc__.strip()
            exit(EXIT_ERROR)
        builder = RingBuilder(int(argv[3]), int(argv[4]), int(argv[5]))
        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
        exit(EXIT_SUCCESS)

    def default():
        """
        Shows information about the ring and the devices within.
        """
        print '%s, build version %d ' % (argv[1], builder.version)
        zones = 0
        balance = 0
        if builder.devs:
            zones = len(set(d['zone'] for d in builder.devs if d is not None))
            #blance = builder.get_balance()
        print '%d partitions, %d replicas, %d zones, %d devices' \
                % (builder.parts, builder.replicas, zones,
                    len([d for d in builder.devs if d]))
        print 'The minimun number of hours before a partition can be' \
                'reassigned is %s' % builder.min_part_hours
        if builder.devs:
            print 'Devices:    id  zone      ip  address  port    name' \
                    'weight partition'
            weighted_parts = builder.parts * builder.replicas / \
                    sum(d['weight'] for d in builder.devs if d is not None)
            for dev in builder.devs:
                if dev is None:
                    continue
                if not dev['weight']:
                    if dev['parts']:
                        blance = 999.99
                    else:
                        blance = 0
                else:
                    blance = 100.0 * dev['parts'] / \
                            (dev['weight'] * weighted_parts) - 100.0
                print '        %5d %5d %15s %5d %9s %6.02f %10s' % \
                        (dev['id'], dev['zone'], dev['ip'], dev['port'],
                         dev['device'], dev['weight'], dev['parts'])
        exit(EXIT_SUCCESS)

    def add():
        """
        .......
        """

        dev_and_weights = izip(islice(argv, 3, len(argv), 2),
                               islice(argv, 4, len(argv), 2))
        for devstr, weightstr in dev_and_weights:
            if not devstr.startswith('z'):
                print 'Invalid add value: %s' % devstr
                exit(EXIT_ERROR)
            i = 1
            while i < len(devstr) and devstr[i].isdigit():
                i += 1
            zone = int(devstr[1:i])
            rest = devstr[i:]

            i = 1
            while i < len(rest) and rest[i] in '0123456789.':
                i += 1
            ip = rest[1:i]
            rest = rest[i:]
            print ip
            i = 1
            while i < len(rest) and rest[i].isdigit():
                i += 1
            port = int(rest[1:i])
            rest = rest[i:]

            i = 1
            while i < len(rest) and rest[i] != '_':
                i += 1
            device_name = rest[1:i]
            rest = rest[i:]

            meta = ''
            if rest.startswith('_'):
                meta = rest[1:]
            weight = float(weightstr)
            for dev in builder.devs:
                if dev is None:
                    continue
                if dev['ip'] == ip and dev['port'] == port and \
                        dev['device'] == device_name:
                    print 'already uses %s'
                    exit(EXIT_ERROR)

            next_dev_id = 0
            if builder.devs:
                next_dev_id = max(d['id'] for d in builder.devs if d) + 1
            builder.add_dev({'id': next_dev_id, 'zone': zone, 'ip': ip,
                             'port': port, 'device': device_name,
                             'weight': weight, 'meta': meta})
            print 'Device z%s-[%s]:%s/%s_"%s" with %s weight got id %s' % \
                    (zone, ip, port, device_name, meta, weight, next_dev_id)
        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
        exit(EXIT_SUCCESS)

    def rebalance():
        """
        Attempts to rebalance the ring by reassigning partitions
        """

        devs_changed = builder.devs_changed
            
        last_balance = builder.get_balance()
        parts, balance = builder.rebalance()
        
        builder.validate()
        print 'Reassigned %d (%.02f%%) partitions. Balance is now %.02f.' % \
                (parts, 100.0*parts / builder.parts, balance)
        status = EXIT_SUCCESS
        if balance > 5:
            print'NOTE: Balance of %.02f indicates you should push the' % \
                    balance
            status = EXIT_WARNING
        builder.get_ring().save(ring_file)
        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
        exit(status)
if __name__ == '__main__':
    
    if exists(argv[1]):
        builder = pickle.load(open(argv[1], 'rb'))
        if not hasattr(builder, 'devs'):
            builder_dict = builder
            builder = RingBuilder(1, 1, 1)
            builder.copy_from(builder_dict)
    ring_file = argv[1]
    if ring_file.endswith('.builder'):
        ring_file = ring_file[:-len('.builder')]
    ring_file += '.ring.gz'
    if len(argv) == 2:
        command = "default"
    else:
        command = argv[2]
    Commands.__dict__.get(command, Commands.unknown)()