diff --git a/bin/swift-ring-builder b/bin/swift-ring-builder index 16859924f5..5779b55c78 100755 --- a/bin/swift-ring-builder +++ b/bin/swift-ring-builder @@ -20,7 +20,7 @@ from errno import EEXIST from itertools import islice, izip from os import mkdir from os.path import basename, abspath, dirname, exists, join as pathjoin -from sys import argv, exit +from sys import argv, exit, stderr from textwrap import wrap from time import time @@ -40,9 +40,11 @@ def format_device(dev): Format a device for display. """ if ':' in dev['ip']: - return 'd%(id)sz%(zone)s-[%(ip)s]:%(port)s/%(device)s_"%(meta)s"' % dev + return ('d%(id)sr%(region)z%(zone)s-' + '[%(ip)s]:%(port)s/%(device)s_"%(meta)s"') % dev else: - return 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_"%(meta)s"' % dev + return ('d%(id)sr%(region)z%(zone)s-' + '%(ip)s:%(port)s/%(device)s_"%(meta)s"') % dev class Commands: @@ -80,19 +82,27 @@ swift-ring-builder Shows information about the ring and the devices within. """ print '%s, build version %d' % (argv[1], builder.version) + regions = 0 zones = 0 balance = 0 + dev_count = 0 if builder.devs: - zones = len(set(d['zone'] for d in builder.devs if d is not None)) + regions = len(set(d['region'] for d in builder.devs + if d is not None)) + zones = len(set((d['region'], d['zone']) for d in builder.devs + if d is not None)) + dev_count = len([d for d in builder.devs + if d is not None]) balance = builder.get_balance() - print '%d partitions, %.6f replicas, %d zones, %d devices, %.02f ' \ - 'balance' % (builder.parts, builder.replicas, zones, - len([d for d in builder.devs if d]), balance) + print '%d partitions, %.6f replicas, %d regions, %d zones, ' \ + '%d devices, %.02f balance' % (builder.parts, builder.replicas, + regions, zones, dev_count, + balance) print 'The minimum number of hours before a partition can be ' \ 'reassigned is %s' % builder.min_part_hours if builder.devs: - print 'Devices: id zone ip address port name ' \ - 'weight partitions balance meta' + print 'Devices: id region zone ip address port' \ + ' name weight partitions balance meta' weighted_parts = builder.parts * builder.replicas / \ sum(d['weight'] for d in builder.devs if d is not None) for dev in builder.devs: @@ -106,10 +116,11 @@ swift-ring-builder else: balance = 100.0 * dev['parts'] / \ (dev['weight'] * weighted_parts) - 100.0 - print ' %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' % \ - (dev['id'], dev['zone'], dev['ip'], dev['port'], - dev['device'], dev['weight'], dev['parts'], balance, - dev['meta']) + print(' %5d %5d %5d %15s %5d %9s %6.02f %10s' + '%7.02f %s' % + (dev['id'], dev['region'], dev['zone'], dev['ip'], + dev['port'], dev['device'], dev['weight'], dev['parts'], + balance, dev['meta'])) exit(EXIT_SUCCESS) def search(): @@ -126,7 +137,7 @@ swift-ring-builder search if not devs: print 'No matching devices found' exit(EXIT_ERROR) - print 'Devices: id zone ip address port name ' \ + print 'Devices: id region zone ip address port name ' \ 'weight partitions balance meta' weighted_parts = builder.parts * builder.replicas / \ sum(d['weight'] for d in builder.devs if d is not None) @@ -139,10 +150,10 @@ swift-ring-builder search else: balance = 100.0 * dev['parts'] / \ (dev['weight'] * weighted_parts) - 100.0 - print ' %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' % \ - (dev['id'], dev['zone'], dev['ip'], dev['port'], - dev['device'], dev['weight'], dev['parts'], balance, - dev['meta']) + print(' %5d %5d %5d %15s %5d %9s %6.02f %10s %7.02f %s' % + (dev['id'], dev['region'], dev['zone'], dev['ip'], + dev['port'], dev['device'], dev['weight'], dev['parts'], + balance, dev['meta'])) exit(EXIT_SUCCESS) def list_parts(): @@ -182,8 +193,8 @@ swift-ring-builder list_parts [] .. def add(): """ swift-ring-builder add - z-:/_ - [z-:/_ ] ... + [r]z-:/_ + [[r]z-:/_ ] ... Adds devices to the ring with the given information. No partitions will be assigned to the new device until after running 'rebalance'. This is so you @@ -196,14 +207,26 @@ swift-ring-builder add devs_and_weights = izip(islice(argv, 3, len(argv), 2), islice(argv, 4, len(argv), 2)) for devstr, weightstr in devs_and_weights: - if not devstr.startswith('z'): + region = 1 + rest = devstr + if devstr.startswith('r'): + i = 1 + while i < len(devstr) and devstr[i].isdigit(): + i += 1 + region = int(devstr[1:i]) + rest = devstr[i:] + else: + stderr.write("WARNING: No region specified for %s. " + "Defaulting to region 1.\n" % devstr) + + if not rest.startswith('z'): print 'Invalid add value: %s' % devstr exit(EXIT_ERROR) i = 1 - while i < len(devstr) and devstr[i].isdigit(): + while i < len(rest) and rest[i].isdigit(): i += 1 - zone = int(devstr[1:i]) - rest = devstr[i:] + zone = int(rest[1:i]) + rest = rest[i:] if not rest.startswith('-'): print 'Invalid add value: %s' % devstr @@ -269,17 +292,21 @@ swift-ring-builder add print "The on-disk ring builder is unchanged.\n" exit(EXIT_ERROR) - builder.add_dev({'zone': zone, 'ip': ip, 'port': port, - 'device': device_name, 'weight': weight, - 'meta': meta}) + builder.add_dev({'region': region, 'zone': zone, 'ip': ip, + 'port': port, 'device': device_name, + 'weight': weight, 'meta': meta}) new_dev = builder.search_devs( - 'z%s-%s:%s/%s' % (zone, ip, port, device_name))[0]['id'] + 'r%dz%d-%s:%s/%s' % + (region, zone, ip, port, device_name))[0]['id'] if ':' in ip: - print 'Device z%s-[%s]:%s/%s_"%s" with %s weight got id %s' % \ - (zone, ip, port, device_name, meta, weight, new_dev) + print( + 'Device r%dz%d-[%s]:%s/%s_"%s" with %s weight got id %s' % + (region, zone, ip, port, + device_name, meta, weight, new_dev)) else: - print 'Device z%s-%s:%s/%s_"%s" with %s weight got id %s' % \ - (zone, ip, port, device_name, meta, weight, new_dev) + print('Device r%dz%d-%s:%s/%s_"%s" with %s weight got id %s' % + (region, zone, ip, port, + device_name, meta, weight, new_dev)) pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_SUCCESS) @@ -442,8 +469,8 @@ swift-ring-builder remove [search-value ...] if len(devs) > 1: print 'Matched more than one device:' for dev in devs: - print ' d%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ - '"%(meta)s"' % dev + print ' d%(id)sr%(region)z%(zone)s-%(ip)s:%(port)s/' \ + '%(device)s_"%(meta)s"' % dev if raw_input('Are you sure you want to remove these %s ' 'devices? (y/N) ' % len(devs)) != 'y': print 'Aborting device removals' @@ -465,9 +492,9 @@ swift-ring-builder remove [search-value ...] print '-' * 79 exit(EXIT_ERROR) - print 'd%(id)sz%(zone)s-%(ip)s:%(port)s/%(device)s_' \ - '"%(meta)s" marked for removal and will be removed' \ - ' next rebalance.' % dev + print 'd%(id)sr%(region)z%(zone)s-%(ip)s:%(port)s/' \ + '%(device)s_"%(meta)s" marked for removal and will ' \ + 'be removed next rebalance.' % dev pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2) exit(EXIT_SUCCESS) diff --git a/doc/source/overview_ring.rst b/doc/source/overview_ring.rst index 9af54959db..3ec163efa3 100644 --- a/doc/source/overview_ring.rst +++ b/doc/source/overview_ring.rst @@ -98,9 +98,9 @@ device and device['weight']]`` Partition Assignment List ************************* -This is a list of array('I') of devices ids. The outermost list contains an -array('I') for each replica. Each array('I') has a length equal to the -partition count for the ring. Each integer in the array('I') is an index into +This is a list of array('H') of devices ids. The outermost list contains an +array('H') for each replica. Each array('H') has a length equal to the +partition count for the ring. Each integer in the array('H') is an index into the above list of devices. The partition list is known internally to the Ring class as _replica2part2dev_id. @@ -108,9 +108,29 @@ So, to create a list of device dictionaries assigned to a partition, the Python code would look like: ``devices = [self.devs[part2dev_id[partition]] for part2dev_id in self._replica2part2dev_id]`` -array('I') is used for memory conservation as there may be millions of +That code is a little simplistic, as it does not account for the +removal of duplicate devices. If a ring has more replicas than +devices, then a partition will have more than one replica on one +device; that's simply the pigeonhole principle at work. + +array('H') is used for memory conservation as there may be millions of partitions. +******************* +Fractional Replicas +******************* + +A ring is not restricted to having an integer number of replicas. In order to +support the gradual changing of replica counts, the ring is able to have a real +number of replicas. + +When the number of replicas is not an integer, then the last element of +_replica2part2dev_id will have a length that is less than the partition count +for the ring. This means that some partitions will have more replicas than +others. For example, if a ring has 3.25 replicas, then 25% of its partitions +will have four replicas, while the remaining 75% will have just three. + + ********************* Partition Shift Value ********************* @@ -123,25 +143,37 @@ in this process. For example, to compute the partition for the path unpack_from('>I', md5('/account/container/object').digest())[0] >> self._part_shift`` +For a ring generated with part_power P, the partition shift value is +32 - P. + ----------------- Building the Ring ----------------- The initial building of the ring first calculates the number of partitions that should ideally be assigned to each device based the device's weight. For -example, if the partition power of 20 the ring will have 1,048,576 partitions. +example, given a partition power of 20, the ring will have 1,048,576 partitions. If there are 1,000 devices of equal weight they will each desire 1,048.576 partitions. The devices are then sorted by the number of partitions they desire and kept in order throughout the initialization process. -Then, the ring builder assigns each replica of each partition to the device -that desires the most partitions at that point while keeping it as far away as +Note: each device is also assigned a random tiebreaker value that is used when +two devices desire the same number of partitions. This tiebreaker is not stored +on disk anywhere, and so two different rings created with the same parameters +will have different partition assignments. For repeatable partition assignments, +``RingBuilder.rebalance()`` takes an optional seed value that will be used to +seed Python's pseudo-random number generator. + +Then, the ring builder assigns each replica of each partition to the device that +desires the most partitions at that point while keeping it as far away as possible from other replicas. The ring builder prefers to assign a replica to a -device in a zone that has no replicas already; should there be no such zone -available, the ring builder will try to find a device on a different server; -failing that, it will just look for a device that has no replicas; finally, if -all other options are exhausted, the ring builder will assign the replica to -the device that has the fewest replicas already assigned. +device in a regions that has no replicas already; should there be no such region +available, the ring builder will try to find a device in a different zone; if +not possible, it will look on a different server; failing that, it will just +look for a device that has no replicas; finally, if all other options are +exhausted, the ring builder will assign the replica to the device that has the +fewest replicas already assigned. Note that assignment of multiple replicas to +one device will only happen if the ring has fewer devices than it has replicas. When building a new ring based on an old ring, the desired number of partitions each device wants is recalculated. Next the partitions to be reassigned are diff --git a/swift/common/ring/builder.py b/swift/common/ring/builder.py index 0e0ac51592..43185789a6 100644 --- a/swift/common/ring/builder.py +++ b/swift/common/ring/builder.py @@ -129,6 +129,11 @@ class RingBuilder(object): self._remove_devs = builder['_remove_devs'] self._ring = None + # Old builders may not have a region defined for their devices, in + # which case we default it to 1. + for dev in self._iter_devs(): + dev.setdefault("region", 1) + def to_dict(self): """ Returns a dict that can be used later with copy_from to @@ -224,9 +229,10 @@ class RingBuilder(object): weight a float of the relative weight of this device as compared to others; this indicates how many partitions the builder will try to assign to this device + region integer indicating which region the device is in zone integer indicating which zone the device is in; a given partition will not be assigned to multiple devices within the - same zone + same (region, zone) pair if there is any alternative ip the ip address of the device port the tcp port of the device device the device's name on disk (sdb1, for example) @@ -413,11 +419,11 @@ class RingBuilder(object): def get_balance(self): """ Get the balance of the ring. The balance value is the highest - percentage off the desired amount of partitions a given device wants. - For instance, if the "worst" device wants (based on its relative weight - and its zone's relative weight) 123 partitions and it has 124 - partitions, the balance value would be 0.83 (1 extra / 123 wanted * 100 - for percentage). + percentage off the desired amount of partitions a given device + wants. For instance, if the "worst" device wants (based on its + weight relative to the sum of all the devices' weights) 123 + partitions and it has 124 partitions, the balance value would + be 0.83 (1 extra / 123 wanted * 100 for percentage). :returns: balance of the ring """ @@ -712,10 +718,10 @@ class RingBuilder(object): they still want and kept in that order throughout the process. The gathered partitions are iterated through, assigning them to devices according to the "most wanted" while keeping the replicas as "far - apart" as possible. Two different zones are considered the - farthest-apart things, followed by different ip/port pairs within a - zone; the least-far-apart things are different devices with the same - ip/port pair in the same zone. + apart" as possible. Two different regions are considered the + farthest-apart things, followed by zones, then different ip/port pairs + within a zone; the least-far-apart things are different devices with + the same ip/port pair in the same zone. If you want more replicas than devices, you won't get all your replicas. @@ -761,8 +767,8 @@ class RingBuilder(object): depth += 1 for part, replace_replicas in reassign_parts: - # Gather up what other tiers (zones, ip_ports, and devices) the - # replicas not-to-be-moved are in for this part. + # Gather up what other tiers (regions, zones, ip/ports, and + # devices) the replicas not-to-be-moved are in for this part. other_replicas = defaultdict(int) unique_tiers_by_tier_len = defaultdict(set) for replica in self._replicas_for_part(part): @@ -977,13 +983,14 @@ class RingBuilder(object): """ The can be of the form:: - dz-:/_ + drz-:/_ Any part is optional, but you must include at least one part. Examples:: d74 Matches the device id 74 + r4 Matches devices in region 4 z1 Matches devices in zone 1 z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4 1.2.3.4 Matches devices in any zone with the ip 1.2.3.4 @@ -997,7 +1004,7 @@ class RingBuilder(object): Most specific example:: - d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8" + d74r4z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8" Nerd explanation: @@ -1012,6 +1019,12 @@ class RingBuilder(object): i += 1 match.append(('id', int(search_value[1:i]))) search_value = search_value[i:] + if search_value.startswith('r'): + i = 1 + while i < len(search_value) and search_value[i].isdigit(): + i += 1 + match.append(('region', int(search_value[1:i]))) + search_value = search_value[i:] if search_value.startswith('z'): i = 1 while i < len(search_value) and search_value[i].isdigit(): diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index a0392b9478..61cc106c47 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -37,6 +37,10 @@ class RingData(object): self._replica2part2dev_id = replica2part2dev_id self._part_shift = part_shift + for dev in self.devs: + if dev is not None: + dev.setdefault("region", 1) + @classmethod def deserialize_v1(cls, gz_file): json_len, = struct.unpack('!I', gz_file.read(4)) @@ -266,39 +270,52 @@ class Ring(object): """ if time() > self._rtime: self._reload() - used = set(part2dev_id[part] - for part2dev_id in self._replica2part2dev_id - if len(part2dev_id) > part) - same_zones = set(self._devs[part2dev_id[part]]['zone'] - for part2dev_id in self._replica2part2dev_id - if len(part2dev_id) > part) + primary_nodes = self._get_part_nodes(part) + + used = set(d['id'] for d in primary_nodes) + same_regions = set(d['region'] for d in primary_nodes) + same_zones = set((d['region'], d['zone']) for d in primary_nodes) + parts = len(self._replica2part2dev_id[0]) start = struct.unpack_from( '>I', md5(str(part)).digest())[0] >> self._part_shift inc = int(parts / 65536) or 1 - # Two loops for execution speed, second loop doesn't need the zone - # check. + # Multiple loops for execution speed; the checks and bookkeeping get + # simpler as you go along for handoff_part in chain(xrange(start, parts, inc), xrange(inc - ((parts - start) % inc), start, inc)): for part2dev_id in self._replica2part2dev_id: - try: + if handoff_part < len(part2dev_id): dev_id = part2dev_id[handoff_part] dev = self._devs[dev_id] - if dev_id not in used and dev['zone'] not in same_zones: + region = dev['region'] + zone = (dev['region'], dev['zone']) + if dev_id not in used and region not in same_regions: yield dev used.add(dev_id) - same_zones.add(dev['zone']) - except IndexError: # Happens with partial replicas - pass + same_regions.add(region) + same_zones.add(zone) + for handoff_part in chain(xrange(start, parts, inc), xrange(inc - ((parts - start) % inc), start, inc)): for part2dev_id in self._replica2part2dev_id: - try: + if handoff_part < len(part2dev_id): + dev_id = part2dev_id[handoff_part] + dev = self._devs[dev_id] + zone = (dev['region'], dev['zone']) + if dev_id not in used and zone not in same_zones: + yield dev + used.add(dev_id) + same_zones.add(zone) + + for handoff_part in chain(xrange(start, parts, inc), + xrange(inc - ((parts - start) % inc), + start, inc)): + for part2dev_id in self._replica2part2dev_id: + if handoff_part < len(part2dev_id): dev_id = part2dev_id[handoff_part] if dev_id not in used: yield self._devs[dev_id] used.add(dev_id) - except IndexError: # Happens with partial replicas - pass diff --git a/swift/common/ring/utils.py b/swift/common/ring/utils.py index 4887b6b874..b99d14d245 100644 --- a/swift/common/ring/utils.py +++ b/swift/common/ring/utils.py @@ -8,13 +8,15 @@ def tiers_for_dev(dev): :returns: tuple of tiers """ - t1 = dev['zone'] - t2 = "{ip}:{port}".format(ip=dev.get('ip'), port=dev.get('port')) - t3 = dev['id'] + t1 = dev['region'] + t2 = dev['zone'] + t3 = "{ip}:{port}".format(ip=dev.get('ip'), port=dev.get('port')) + t4 = dev['id'] return ((t1,), (t1, t2), - (t1, t2, t3)) + (t1, t2, t3), + (t1, t2, t3, t4)) def build_tier_tree(devices): @@ -27,52 +29,72 @@ def build_tier_tree(devices): Example: - zone 1 -+---- 192.168.1.1:6000 -+---- device id 0 - | | - | +---- device id 1 - | | - | +---- device id 2 - | - +---- 192.168.1.2:6000 -+---- device id 3 - | - +---- device id 4 - | - +---- device id 5 + region 1 -+---- zone 1 -+---- 192.168.101.1:6000 -+---- device id 0 + | | | + | | +---- device id 1 + | | | + | | +---- device id 2 + | | + | +---- 192.168.101.2:6000 -+---- device id 3 + | | + | +---- device id 4 + | | + | +---- device id 5 + | + +---- zone 2 -+---- 192.168.102.1:6000 -+---- device id 6 + | | + | +---- device id 7 + | | + | +---- device id 8 + | + +---- 192.168.102.2:6000 -+---- device id 9 + | + +---- device id 10 - zone 2 -+---- 192.168.2.1:6000 -+---- device id 6 - | | - | +---- device id 7 - | | - | +---- device id 8 - | - +---- 192.168.2.2:6000 -+---- device id 9 - | - +---- device id 10 - | - +---- device id 11 + region 2 -+---- zone 1 -+---- 192.168.201.1:6000 -+---- device id 12 + | | + | +---- device id 13 + | | + | +---- device id 14 + | + +---- 192.168.201.2:6000 -+---- device id 15 + | + +---- device id 16 + | + +---- device id 17 The tier tree would look like: { (): [(1,), (2,)], - (1,): [(1, 192.168.1.1:6000), - (1, 192.168.1.2:6000)], - (2,): [(2, 192.168.2.1:6000), - (2, 192.168.2.2:6000)], + (1,): [(1, 1), (1, 2)], + (2,): [(2, 1)], - (1, 192.168.1.1:6000): [(1, 192.168.1.1:6000, 0), - (1, 192.168.1.1:6000, 1), - (1, 192.168.1.1:6000, 2)], - (1, 192.168.1.2:6000): [(1, 192.168.1.2:6000, 3), - (1, 192.168.1.2:6000, 4), - (1, 192.168.1.2:6000, 5)], - (2, 192.168.2.1:6000): [(2, 192.168.2.1:6000, 6), - (2, 192.168.2.1:6000, 7), - (2, 192.168.2.1:6000, 8)], - (2, 192.168.2.2:6000): [(2, 192.168.2.2:6000, 9), - (2, 192.168.2.2:6000, 10), - (2, 192.168.2.2:6000, 11)], + (1, 1): [(1, 1, 192.168.101.1:6000), + (1, 1, 192.168.101.2:6000)], + (1, 2): [(1, 2, 192.168.102.1:6000), + (1, 2, 192.168.102.2:6000)], + (2, 1): [(2, 1, 192.168.201.1:6000), + (2, 1, 192.168.201.2:6000)], + + (1, 1, 192.168.101.1:6000): [(1, 1, 192.168.101.1:6000, 0), + (1, 1, 192.168.101.1:6000, 1), + (1, 1, 192.168.101.1:6000, 2)], + (1, 1, 192.168.101.2:6000): [(1, 1, 192.168.101.2:6000, 3), + (1, 1, 192.168.101.2:6000, 4), + (1, 1, 192.168.101.2:6000, 5)], + (1, 2, 192.168.102.1:6000): [(1, 2, 192.168.102.1:6000, 6), + (1, 2, 192.168.102.1:6000, 7), + (1, 2, 192.168.102.1:6000, 8)], + (1, 2, 192.168.102.2:6000): [(1, 2, 192.168.102.2:6000, 9), + (1, 2, 192.168.102.2:6000, 10)], + (2, 1, 192.168.201.1:6000): [(2, 1, 192.168.201.1:6000, 12), + (2, 1, 192.168.201.1:6000, 13), + (2, 1, 192.168.201.1:6000, 14)], + (2, 1, 192.168.201.2:6000): [(2, 1, 192.168.201.2:6000, 15), + (2, 1, 192.168.201.2:6000, 16), + (2, 1, 192.168.201.2:6000, 17)], } :devices: device dicts from which to generate the tree diff --git a/test/unit/common/ring/test_builder.py b/test/unit/common/ring/test_builder.py index 2e6ab58397..5a1d4faf10 100644 --- a/test/unit/common/ring/test_builder.py +++ b/test/unit/common/ring/test_builder.py @@ -49,14 +49,14 @@ class TestRingBuilder(unittest.TestCase): def test_get_ring(self): rb = ring.RingBuilder(8, 3, 1) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sda1'}) - rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10002, 'device': 'sda1'}) - rb.add_dev({'id': 3, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10004, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) + rb.add_dev({'id': 3, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'}) rb.remove_dev(1) rb.rebalance() r = rb.get_ring() @@ -75,7 +75,7 @@ class TestRingBuilder(unittest.TestCase): for n in range(3): rb = ring.RingBuilder(8, 3, 1) for idx, (zone, port) in enumerate(devs): - rb.add_dev({'id': idx, 'zone': zone, 'weight': 1, + rb.add_dev({'id': idx, 'region': 0, 'zone': zone, 'weight': 1, 'ip': '127.0.0.1', 'port': port, 'device': 'sda1'}) ring_builders.append(rb) @@ -110,28 +110,30 @@ class TestRingBuilder(unittest.TestCase): def test_add_dev(self): rb = ring.RingBuilder(8, 3, 1) - dev = \ - {'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000} + dev = {'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000} rb.add_dev(dev) self.assertRaises(exceptions.DuplicateDeviceError, rb.add_dev, dev) rb = ring.RingBuilder(8, 3, 1) #test add new dev with no id - rb.add_dev({'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 6000}) + rb.add_dev({'zone': 0, 'region': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 6000}) self.assertEquals(rb.devs[0]['id'], 0) #test add another dev with no id - rb.add_dev({'zone': 3, 'weight': 1, 'ip': '127.0.0.1', 'port': 6000}) + rb.add_dev({'zone': 3, 'region': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 6000}) self.assertEquals(rb.devs[1]['id'], 1) def test_set_dev_weight(self): rb = ring.RingBuilder(8, 3, 1) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 0.5, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'zone': 0, 'weight': 0.5, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sda1'}) - rb.add_dev({'id': 2, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10002, 'device': 'sda1'}) - rb.add_dev({'id': 3, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10003, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 0.5, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 0.5, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) + rb.add_dev({'id': 3, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) rb.rebalance() r = rb.get_ring() counts = {} @@ -152,14 +154,14 @@ class TestRingBuilder(unittest.TestCase): def test_remove_dev(self): rb = ring.RingBuilder(8, 3, 1) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sda1'}) - rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10002, 'device': 'sda1'}) - rb.add_dev({'id': 3, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10003, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) + rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) rb.rebalance() r = rb.get_ring() counts = {} @@ -180,17 +182,17 @@ class TestRingBuilder(unittest.TestCase): def test_remove_a_lot(self): rb = ring.RingBuilder(3, 3, 1) rb.add_dev({'id': 0, 'device': 'd0', 'ip': '10.0.0.1', - 'port': 6002, 'weight': 1000.0, 'zone': 1}) + 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 1}) rb.add_dev({'id': 1, 'device': 'd1', 'ip': '10.0.0.2', - 'port': 6002, 'weight': 1000.0, 'zone': 2}) + 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 2}) rb.add_dev({'id': 2, 'device': 'd2', 'ip': '10.0.0.3', - 'port': 6002, 'weight': 1000.0, 'zone': 3}) + 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 3}) rb.add_dev({'id': 3, 'device': 'd3', 'ip': '10.0.0.1', - 'port': 6002, 'weight': 1000.0, 'zone': 1}) + 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 1}) rb.add_dev({'id': 4, 'device': 'd4', 'ip': '10.0.0.2', - 'port': 6002, 'weight': 1000.0, 'zone': 2}) + 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 2}) rb.add_dev({'id': 5, 'device': 'd5', 'ip': '10.0.0.3', - 'port': 6002, 'weight': 1000.0, 'zone': 3}) + 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 3}) rb.rebalance() rb.validate() @@ -216,15 +218,15 @@ class TestRingBuilder(unittest.TestCase): def _shuffled_gather_helper(self): rb = ring.RingBuilder(8, 3, 1) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sda1'}) - rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10002, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) rb.rebalance() - rb.add_dev({'id': 3, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10003, 'device': 'sda1'}) + rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) rb.pretend_min_part_hours_passed() parts = rb._gather_reassign_parts() max_run = 0 @@ -243,34 +245,64 @@ class TestRingBuilder(unittest.TestCase): return max_run > len(parts) / 2 def test_multitier_partial(self): - # Multitier test, zones full, nodes not full - rb = ring.RingBuilder(8, 6, 1) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda'}) - rb.add_dev({'id': 1, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdb'}) - rb.add_dev({'id': 2, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdc'}) - - rb.add_dev({'id': 3, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sdd'}) - rb.add_dev({'id': 4, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sde'}) - rb.add_dev({'id': 5, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sdf'}) - - rb.add_dev({'id': 6, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10002, 'device': 'sdg'}) - rb.add_dev({'id': 7, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10002, 'device': 'sdh'}) - rb.add_dev({'id': 8, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10002, 'device': 'sdi'}) + # Multitier test, nothing full + rb = ring.RingBuilder(8, 3, 1) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'}) + rb.add_dev({'id': 1, 'region': 1, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'}) + rb.add_dev({'id': 2, 'region': 2, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'}) + rb.add_dev({'id': 3, 'region': 3, 'zone': 3, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdd'}) rb.rebalance() rb.validate() for part in xrange(rb.parts): - counts = defaultdict(lambda: defaultdict(lambda: 0)) + counts = defaultdict(lambda: defaultdict(int)) + for replica in xrange(rb.replicas): + dev = rb.devs[rb._replica2part2dev[replica][part]] + counts['region'][dev['region']] += 1 + counts['zone'][dev['zone']] += 1 + + if any(c > 1 for c in counts['region'].values()): + raise AssertionError( + "Partition %d not evenly region-distributed (got %r)" % + (part, counts['region'])) + if any(c > 1 for c in counts['zone'].values()): + raise AssertionError( + "Partition %d not evenly zone-distributed (got %r)" % + (part, counts['zone'])) + + # Multitier test, zones full, nodes not full + rb = ring.RingBuilder(8, 6, 1) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'}) + + rb.add_dev({'id': 3, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdd'}) + rb.add_dev({'id': 4, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sde'}) + rb.add_dev({'id': 5, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdf'}) + + rb.add_dev({'id': 6, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10002, 'device': 'sdg'}) + rb.add_dev({'id': 7, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10002, 'device': 'sdh'}) + rb.add_dev({'id': 8, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10002, 'device': 'sdi'}) + + rb.rebalance() + rb.validate() + + for part in xrange(rb.parts): + counts = defaultdict(lambda: defaultdict(int)) for replica in xrange(rb.replicas): dev = rb.devs[rb._replica2part2dev[replica][part]] counts['zone'][dev['zone']] += 1 @@ -288,26 +320,26 @@ class TestRingBuilder(unittest.TestCase): def test_multitier_full(self): # Multitier test, #replicas == #devs rb = ring.RingBuilder(8, 6, 1) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda'}) - rb.add_dev({'id': 1, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdb'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'}) - rb.add_dev({'id': 2, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdc'}) - rb.add_dev({'id': 3, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sdd'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'}) + rb.add_dev({'id': 3, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdd'}) - rb.add_dev({'id': 4, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sde'}) - rb.add_dev({'id': 5, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sdf'}) + rb.add_dev({'id': 4, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sde'}) + rb.add_dev({'id': 5, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdf'}) rb.rebalance() rb.validate() for part in xrange(rb.parts): - counts = defaultdict(lambda: defaultdict(lambda: 0)) + counts = defaultdict(lambda: defaultdict(int)) for replica in xrange(rb.replicas): dev = rb.devs[rb._replica2part2dev[replica][part]] counts['zone'][dev['zone']] += 1 @@ -325,26 +357,26 @@ class TestRingBuilder(unittest.TestCase): def test_multitier_overfull(self): # Multitier test, #replicas > #devs + 2 (to prove even distribution) rb = ring.RingBuilder(8, 8, 1) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda'}) - rb.add_dev({'id': 1, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdb'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'}) - rb.add_dev({'id': 2, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdc'}) - rb.add_dev({'id': 3, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sdd'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'}) + rb.add_dev({'id': 3, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdd'}) - rb.add_dev({'id': 4, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sde'}) - rb.add_dev({'id': 5, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sdf'}) + rb.add_dev({'id': 4, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sde'}) + rb.add_dev({'id': 5, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdf'}) rb.rebalance() rb.validate() for part in xrange(rb.parts): - counts = defaultdict(lambda: defaultdict(lambda: 0)) + counts = defaultdict(lambda: defaultdict(int)) for replica in xrange(rb.replicas): dev = rb.devs[rb._replica2part2dev[replica][part]] counts['zone'][dev['zone']] += 1 @@ -365,22 +397,22 @@ class TestRingBuilder(unittest.TestCase): def test_multitier_expansion_more_devices(self): rb = ring.RingBuilder(8, 6, 1) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda'}) - rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdb'}) - rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdc'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'}) rb.rebalance() rb.validate() - rb.add_dev({'id': 3, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdd'}) - rb.add_dev({'id': 4, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sde'}) - rb.add_dev({'id': 5, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdf'}) + rb.add_dev({'id': 3, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdd'}) + rb.add_dev({'id': 4, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sde'}) + rb.add_dev({'id': 5, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdf'}) for _ in xrange(5): rb.pretend_min_part_hours_passed() @@ -388,8 +420,8 @@ class TestRingBuilder(unittest.TestCase): rb.validate() for part in xrange(rb.parts): - counts = dict(zone=defaultdict(lambda: 0), - dev_id=defaultdict(lambda: 0)) + counts = dict(zone=defaultdict(int), + dev_id=defaultdict(int)) for replica in xrange(rb.replicas): dev = rb.devs[rb._replica2part2dev[replica][part]] counts['zone'][dev['zone']] += 1 @@ -401,17 +433,17 @@ class TestRingBuilder(unittest.TestCase): def test_multitier_part_moves_with_0_min_part_hours(self): rb = ring.RingBuilder(8, 3, 0) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.rebalance() rb.validate() # min_part_hours is 0, so we're clear to move 2 replicas to # new devs - rb.add_dev({'id': 1, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdb1'}) - rb.add_dev({'id': 2, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdc1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb1'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc1'}) rb.rebalance() rb.validate() @@ -426,17 +458,17 @@ class TestRingBuilder(unittest.TestCase): def test_multitier_part_moves_with_positive_min_part_hours(self): rb = ring.RingBuilder(8, 3, 99) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.rebalance() rb.validate() # min_part_hours is >0, so we'll only be able to move 1 # replica to a new home - rb.add_dev({'id': 1, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdb1'}) - rb.add_dev({'id': 2, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdc1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb1'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc1'}) rb.pretend_min_part_hours_passed() rb.rebalance() rb.validate() @@ -453,20 +485,20 @@ class TestRingBuilder(unittest.TestCase): def test_multitier_dont_move_too_many_replicas(self): rb = ring.RingBuilder(8, 3, 0) # there'll be at least one replica in z0 and z1 - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdb1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb1'}) rb.rebalance() rb.validate() # only 1 replica should move - rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdd1'}) - rb.add_dev({'id': 3, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sde1'}) - rb.add_dev({'id': 4, 'zone': 4, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sdf1'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdd1'}) + rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sde1'}) + rb.add_dev({'id': 4, 'region': 0, 'zone': 4, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdf1'}) rb.rebalance() rb.validate() @@ -485,12 +517,12 @@ class TestRingBuilder(unittest.TestCase): def test_rerebalance(self): rb = ring.RingBuilder(8, 3, 1) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sda1'}) - rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10002, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) rb.rebalance() r = rb.get_ring() counts = {} @@ -498,8 +530,8 @@ class TestRingBuilder(unittest.TestCase): for dev_id in part2dev_id: counts[dev_id] = counts.get(dev_id, 0) + 1 self.assertEquals(counts, {0: 256, 1: 256, 2: 256}) - rb.add_dev({'id': 3, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10003, 'device': 'sda1'}) + rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) rb.pretend_min_part_hours_passed() rb.rebalance() r = rb.get_ring() @@ -521,21 +553,21 @@ class TestRingBuilder(unittest.TestCase): """ Test for https://bugs.launchpad.net/swift/+bug/845952 """ # min_part of 0 to allow for rapid rebalancing rb = ring.RingBuilder(8, 3, 0) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sda1'}) - rb.add_dev({'id': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10002, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) rb.rebalance() - rb.add_dev({'id': 3, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10003, 'device': 'sda1'}) - rb.add_dev({'id': 4, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10004, 'device': 'sda1'}) - rb.add_dev({'id': 5, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10005, 'device': 'sda1'}) + rb.add_dev({'id': 3, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) + rb.add_dev({'id': 4, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'}) + rb.add_dev({'id': 5, 'region': 0, 'zone': 2, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10005, 'device': 'sda1'}) rb.rebalance() @@ -545,10 +577,10 @@ class TestRingBuilder(unittest.TestCase): def test_set_replicas_increase(self): rb = ring.RingBuilder(8, 2, 0) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.rebalance() rb.validate() @@ -567,10 +599,10 @@ class TestRingBuilder(unittest.TestCase): def test_set_replicas_decrease(self): rb = ring.RingBuilder(4, 5, 0) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.rebalance() rb.validate() @@ -593,10 +625,10 @@ class TestRingBuilder(unittest.TestCase): def test_fractional_replicas_rebalance(self): rb = ring.RingBuilder(8, 2.5, 0) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 0, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.rebalance() # passes by not crashing rb.validate() # also passes by not crashing self.assertEqual([len(p2d) for p2d in rb._replica2part2dev], @@ -604,14 +636,17 @@ class TestRingBuilder(unittest.TestCase): def test_load(self): rb = ring.RingBuilder(8, 3, 1) - devs = [{'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0', - 'port': 10000, 'device': 'sda1', 'meta': 'meta0'}, - {'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sdb1', 'meta': 'meta1'}, - {'id': 2, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2', - 'port': 10002, 'device': 'sdc1', 'meta': 'meta2'}, - {'id': 3, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3', - 'port': 10003, 'device': 'sdd1'}] + devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.0', 'port': 10000, 'device': 'sda1', + 'meta': 'meta0'}, + {'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdb1', + 'meta': 'meta1'}, + {'id': 2, 'region': 0, 'zone': 2, 'weight': 2, + 'ip': '127.0.0.2', 'port': 10002, 'device': 'sdc1', + 'meta': 'meta2'}, + {'id': 3, 'region': 0, 'zone': 3, 'weight': 2, + 'ip': '127.0.0.3', 'port': 10003, 'device': 'sdd1'}] for d in devs: rb.add_dev(d) rb.rebalance() @@ -653,17 +688,27 @@ class TestRingBuilder(unittest.TestCase): def test_search_devs(self): rb = ring.RingBuilder(8, 3, 1) - devs = [{'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0', - 'port': 10000, 'device': 'sda1', 'meta': 'meta0'}, - {'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sdb1', 'meta': 'meta1'}, - {'id': 2, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2', - 'port': 10002, 'device': 'sdc1', 'meta': 'meta2'}, - {'id': 3, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3', - 'port': 10003, 'device': 'sdd1', 'meta': 'meta3'}] + devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.0', 'port': 10000, 'device': 'sda1', + 'meta': 'meta0'}, + {'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdb1', + 'meta': 'meta1'}, + {'id': 2, 'region': 1, 'zone': 2, 'weight': 2, + 'ip': '127.0.0.2', 'port': 10002, 'device': 'sdc1', + 'meta': 'meta2'}, + {'id': 3, 'region': 1, 'zone': 3, 'weight': 2, + 'ip': '127.0.0.3', 'port': 10003, 'device': 'sdffd1', + 'meta': 'meta3'}] for d in devs: rb.add_dev(d) rb.rebalance() + res = rb.search_devs('r0') + self.assertEquals(res, [devs[0], devs[1]]) + res = rb.search_devs('r1') + self.assertEquals(res, [devs[2], devs[3]]) + res = rb.search_devs('r1z2') + self.assertEquals(res, [devs[2]]) res = rb.search_devs('d1') self.assertEquals(res, [devs[1]]) res = rb.search_devs('z1') @@ -681,14 +726,14 @@ class TestRingBuilder(unittest.TestCase): def test_validate(self): rb = ring.RingBuilder(8, 3, 1) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sda1'}) - rb.add_dev({'id': 2, 'zone': 2, 'weight': 2, 'ip': '127.0.0.1', - 'port': 10002, 'device': 'sda1'}) - rb.add_dev({'id': 3, 'zone': 3, 'weight': 2, 'ip': '127.0.0.1', - 'port': 10003, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 2, + 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) + rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 2, + 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) rb.rebalance() r = rb.get_ring() counts = {} @@ -744,18 +789,18 @@ class TestRingBuilder(unittest.TestCase): # Validate that zero weight devices with no partitions don't count on # the 'worst' value. self.assertNotEquals(rb.validate(stats=True)[1], 999.99) - rb.add_dev({'id': 4, 'zone': 0, 'weight': 0, 'ip': '127.0.0.1', - 'port': 10004, 'device': 'sda1'}) + rb.add_dev({'id': 4, 'region': 0, 'zone': 0, 'weight': 0, + 'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'}) rb.pretend_min_part_hours_passed() rb.rebalance() self.assertNotEquals(rb.validate(stats=True)[1], 999.99) def test_get_part_devices(self): rb = ring.RingBuilder(8, 3, 1) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.rebalance() part_devs = sorted(rb.get_part_devices(0), @@ -764,10 +809,10 @@ class TestRingBuilder(unittest.TestCase): def test_get_part_devices_partial_replicas(self): rb = ring.RingBuilder(8, 2.5, 1) - rb.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', - 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.rebalance() # note: partition 255 will only have 2 replicas diff --git a/test/unit/common/ring/test_ring.py b/test/unit/common/ring/test_ring.py index c2fdf84d44..00606d5666 100644 --- a/test/unit/common/ring/test_ring.py +++ b/test/unit/common/ring/test_ring.py @@ -43,7 +43,8 @@ class TestRingData(unittest.TestCase): def test_attrs(self): r2p2d = [[0, 1, 0, 1], [0, 1, 0, 1]] - d = [{'id': 0, 'zone': 0}, {'id': 1, 'zone': 1}] + d = [{'id': 0, 'zone': 0, 'region': 0}, + {'id': 1, 'zone': 1, 'region': 1}] s = 30 rd = ring.RingData(r2p2d, d, s) self.assertEquals(rd._replica2part2dev_id, r2p2d) @@ -104,14 +105,14 @@ class TestRing(unittest.TestCase): array.array('H', [0, 1, 0, 1]), array.array('H', [0, 1, 0, 1]), array.array('H', [3, 4, 3, 4])] - self.intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0, + self.intended_devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000}, - {'id': 1, 'zone': 0, 'weight': 1.0, + {'id': 1, 'region': 0, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000}, None, - {'id': 3, 'zone': 2, 'weight': 1.0, + {'id': 3, 'region': 0, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.1', 'port': 6000}, - {'id': 4, 'zone': 2, 'weight': 1.0, + {'id': 4, 'region': 0, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.2', 'port': 6000}] self.intended_part_shift = 30 self.intended_reload_time = 15 @@ -149,7 +150,7 @@ class TestRing(unittest.TestCase): ring_name='whatever') orig_mtime = self.ring._mtime self.assertEquals(len(self.ring.devs), 5) - self.intended_devs.append({'id': 3, 'zone': 3, 'weight': 1.0}) + self.intended_devs.append({'id': 3, 'region': 0, 'zone': 3, 'weight': 1.0}) ring.RingData(self.intended_replica2part2dev_id, self.intended_devs, self.intended_part_shift).save(self.testgz) sleep(0.1) @@ -162,7 +163,7 @@ class TestRing(unittest.TestCase): ring_name='whatever') orig_mtime = self.ring._mtime self.assertEquals(len(self.ring.devs), 6) - self.intended_devs.append({'id': 5, 'zone': 4, 'weight': 1.0}) + self.intended_devs.append({'id': 5, 'region': 0, 'zone': 4, 'weight': 1.0}) ring.RingData(self.intended_replica2part2dev_id, self.intended_devs, self.intended_part_shift).save(self.testgz) sleep(0.1) @@ -176,7 +177,7 @@ class TestRing(unittest.TestCase): orig_mtime = self.ring._mtime part, nodes = self.ring.get_nodes('a') self.assertEquals(len(self.ring.devs), 7) - self.intended_devs.append({'id': 6, 'zone': 5, 'weight': 1.0}) + self.intended_devs.append({'id': 6, 'region': 0, 'zone': 5, 'weight': 1.0}) ring.RingData(self.intended_replica2part2dev_id, self.intended_devs, self.intended_part_shift).save(self.testgz) sleep(0.1) @@ -189,7 +190,7 @@ class TestRing(unittest.TestCase): ring_name='whatever') orig_mtime = self.ring._mtime self.assertEquals(len(self.ring.devs), 8) - self.intended_devs.append({'id': 5, 'zone': 4, 'weight': 1.0}) + self.intended_devs.append({'id': 5, 'region': 0, 'zone': 4, 'weight': 1.0}) ring.RingData(self.intended_replica2part2dev_id, self.intended_devs, self.intended_part_shift).save(self.testgz) sleep(0.1) @@ -312,7 +313,8 @@ class TestRing(unittest.TestCase): for device in xrange(1, 4): rb.add_dev({'id': next_dev_id, 'ip': '1.2.%d.%d' % (zone, server), - 'port': 1234, 'zone': zone, 'weight': 1.0}) + 'port': 1234, 'zone': zone, 'region': 0, + 'weight': 1.0}) next_dev_id += 1 rb.rebalance(seed=1) rb.get_ring().save(self.testgz) @@ -343,7 +345,7 @@ class TestRing(unittest.TestCase): server = 0 rb.add_dev({'id': next_dev_id, 'ip': '1.2.%d.%d' % (zone, server), - 'port': 1234, 'zone': zone, 'weight': 1.0}) + 'port': 1234, 'zone': zone, 'region': 0, 'weight': 1.0}) next_dev_id += 1 rb.rebalance(seed=1) rb.get_ring().save(self.testgz) @@ -542,6 +544,57 @@ class TestRing(unittest.TestCase): seen_zones.update(d['zone'] for d in devs2[:6]) self.assertEquals(seen_zones, set(range(1, 10))) + # Test distribution across regions + rb.set_replicas(3) + for region in xrange(1, 5): + rb.add_dev({'id': next_dev_id, + 'ip': '1.%d.1.%d' % (region, server), 'port': 1234, + 'zone': 1, 'region': region, 'weight': 1.0}) + next_dev_id += 1 + rb.pretend_min_part_hours_passed() + rb.rebalance(seed=1) + rb.pretend_min_part_hours_passed() + rb.rebalance(seed=1) + rb.get_ring().save(self.testgz) + r = ring.Ring(self.testdir, ring_name='whatever') + + # There's 5 regions now, so the primary nodes + first 2 handoffs + # should span all 5 regions + part, devs = r.get_nodes('a1', 'c1', 'o1') + primary_regions = set(d['region'] for d in devs) + primary_zones = set((d['region'], d['zone']) for d in devs) + more_devs = list(r.get_more_nodes(part)) + + seen_regions = set(primary_regions) + seen_regions.update(d['region'] for d in more_devs[:2]) + self.assertEquals(seen_regions, set(range(0, 5))) + + # There are 13 zones now, so the first 13 nodes should all have + # distinct zones (that's r0z0, r0z1, ..., r0z8, r1z1, r2z1, r3z1, and + # r4z1). + seen_zones = set(primary_zones) + seen_zones.update((d['region'], d['zone']) for d in more_devs[:10]) + self.assertEquals(13, len(seen_zones)) + + # Here's a brittle canary-in-the-coalmine test to make sure the region + # handoff computation didn't change accidentally + exp_handoffs = [111, 112, 74, 54, 93, 31, 2, 43, 100, 22, 71, 32, 92, + 35, 9, 50, 41, 76, 80, 84, 88, 17, 94, 101, 1, 10, 96, + 44, 73, 6, 75, 102, 37, 21, 97, 29, 105, 5, 28, 47, + 106, 30, 16, 39, 77, 42, 72, 20, 13, 34, 99, 108, 14, + 66, 61, 81, 90, 4, 40, 3, 45, 62, 7, 15, 87, 12, 83, + 89, 53, 33, 98, 49, 65, 25, 107, 56, 58, 86, 48, 57, + 24, 11, 23, 26, 46, 64, 69, 38, 36, 79, 63, 104, 51, + 70, 82, 67, 68, 8, 95, 91, 55, 59, 85] + dev_ids = [d['id'] for d in more_devs] + + self.assertEquals(len(dev_ids), len(exp_handoffs)) + for index, dev_id in enumerate(dev_ids): + self.assertEquals( + dev_id, exp_handoffs[index], + 'handoff differs at position %d\n%s\n%s' % ( + index, dev_ids[index:], exp_handoffs[index:])) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/ring/test_utils.py b/test/unit/common/ring/test_utils.py index 273f144e07..10a2834a7c 100644 --- a/test/unit/common/ring/test_utils.py +++ b/test/unit/common/ring/test_utils.py @@ -21,22 +21,34 @@ from swift.common.ring.utils import build_tier_tree, tiers_for_dev class TestUtils(unittest.TestCase): def setUp(self): - self.test_dev = {'zone': 1, 'ip': '192.168.1.1', + self.test_dev = {'region': 1, 'zone': 1, 'ip': '192.168.1.1', 'port': '6000', 'id': 0} def get_test_devs(): - dev0 = {'zone': 1, 'ip': '192.168.1.1', 'port': '6000', 'id': 0} - dev1 = {'zone': 1, 'ip': '192.168.1.1', 'port': '6000', 'id': 1} - dev2 = {'zone': 1, 'ip': '192.168.1.1', 'port': '6000', 'id': 2} - dev3 = {'zone': 1, 'ip': '192.168.1.2', 'port': '6000', 'id': 3} - dev4 = {'zone': 1, 'ip': '192.168.1.2', 'port': '6000', 'id': 4} - dev5 = {'zone': 1, 'ip': '192.168.1.2', 'port': '6000', 'id': 5} - dev6 = {'zone': 2, 'ip': '192.168.2.1', 'port': '6000', 'id': 6} - dev7 = {'zone': 2, 'ip': '192.168.2.1', 'port': '6000', 'id': 7} - dev8 = {'zone': 2, 'ip': '192.168.2.1', 'port': '6000', 'id': 8} - dev9 = {'zone': 2, 'ip': '192.168.2.2', 'port': '6000', 'id': 9} - dev10 = {'zone': 2, 'ip': '192.168.2.2', 'port': '6000', 'id': 10} - dev11 = {'zone': 2, 'ip': '192.168.2.2', 'port': '6000', 'id': 11} + dev0 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1', + 'port': '6000', 'id': 0} + dev1 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1', + 'port': '6000', 'id': 1} + dev2 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1', + 'port': '6000', 'id': 2} + dev3 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2', + 'port': '6000', 'id': 3} + dev4 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2', + 'port': '6000', 'id': 4} + dev5 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2', + 'port': '6000', 'id': 5} + dev6 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1', + 'port': '6000', 'id': 6} + dev7 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1', + 'port': '6000', 'id': 7} + dev8 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1', + 'port': '6000', 'id': 8} + dev9 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2', + 'port': '6000', 'id': 9} + dev10 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2', + 'port': '6000', 'id': 10} + dev11 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2', + 'port': '6000', 'id': 11} return [dev0, dev1, dev2, dev3, dev4, dev5, dev6, dev7, dev8, dev9, dev10, dev11] @@ -44,34 +56,38 @@ class TestUtils(unittest.TestCase): def test_tiers_for_dev(self): self.assertEqual(tiers_for_dev(self.test_dev), - ((1,), (1, '192.168.1.1:6000'), (1, '192.168.1.1:6000', 0))) + ((1,), + (1, 1), + (1, 1, '192.168.1.1:6000'), + (1, 1, '192.168.1.1:6000', 0))) def test_build_tier_tree(self): ret = build_tier_tree(self.test_devs) - self.assertEqual(len(ret), 7) - self.assertEqual(ret[()], set([(2,), (1,)])) - self.assertEqual(ret[(1,)], - set([(1, '192.168.1.2:6000'), - (1, '192.168.1.1:6000')])) - self.assertEqual(ret[(2,)], - set([(2, '192.168.2.2:6000'), - (2, '192.168.2.1:6000')])) - self.assertEqual(ret[(1, '192.168.1.1:6000')], - set([(1, '192.168.1.1:6000', 0), - (1, '192.168.1.1:6000', 1), - (1, '192.168.1.1:6000', 2)])) - self.assertEqual(ret[(1, '192.168.1.2:6000')], - set([(1, '192.168.1.2:6000', 3), - (1, '192.168.1.2:6000', 4), - (1, '192.168.1.2:6000', 5)])) - self.assertEqual(ret[(2, '192.168.2.1:6000')], - set([(2, '192.168.2.1:6000', 6), - (2, '192.168.2.1:6000', 7), - (2, '192.168.2.1:6000', 8)])) - self.assertEqual(ret[(2, '192.168.2.2:6000')], - set([(2, '192.168.2.2:6000', 9), - (2, '192.168.2.2:6000', 10), - (2, '192.168.2.2:6000', 11)])) + self.assertEqual(len(ret), 8) + self.assertEqual(ret[()], set([(1,)])) + self.assertEqual(ret[(1,)], set([(1, 1), (1, 2)])) + self.assertEqual(ret[(1, 1)], + set([(1, 1, '192.168.1.2:6000'), + (1, 1, '192.168.1.1:6000')])) + self.assertEqual(ret[(1, 2)], + set([(1, 2, '192.168.2.2:6000'), + (1, 2, '192.168.2.1:6000')])) + self.assertEqual(ret[(1, 1, '192.168.1.1:6000')], + set([(1, 1, '192.168.1.1:6000', 0), + (1, 1, '192.168.1.1:6000', 1), + (1, 1, '192.168.1.1:6000', 2)])) + self.assertEqual(ret[(1, 1, '192.168.1.2:6000')], + set([(1, 1, '192.168.1.2:6000', 3), + (1, 1, '192.168.1.2:6000', 4), + (1, 1, '192.168.1.2:6000', 5)])) + self.assertEqual(ret[(1, 2, '192.168.2.1:6000')], + set([(1, 2, '192.168.2.1:6000', 6), + (1, 2, '192.168.2.1:6000', 7), + (1, 2, '192.168.2.1:6000', 8)])) + self.assertEqual(ret[(1, 2, '192.168.2.2:6000')], + set([(1, 2, '192.168.2.2:6000', 9), + (1, 2, '192.168.2.2:6000', 10), + (1, 2, '192.168.2.2:6000', 11)])) if __name__ == '__main__':