diff --git a/doc/source/overview_ring.rst b/doc/source/overview_ring.rst index 6ed399afc4..118a437788 100644 --- a/doc/source/overview_ring.rst +++ b/doc/source/overview_ring.rst @@ -75,7 +75,7 @@ weight float The relative weight of the device in comparison to other back into balance a device that has ended up with more or less data than desired over time. A good average weight of 100.0 allows flexibility in lowering the weight later if necessary. -ip string The IP address of the server containing the device. +ip string The IP address or hostname of the server containing the device. port int The TCP port the listening server process uses that serves requests for the device. device string The on disk name of the device on the server. diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 0891d6fc59..c8d623691a 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -28,6 +28,7 @@ from swift.common.direct_client import direct_delete_container, \ direct_delete_object, direct_get_container from swift.common.exceptions import ClientException from swift.common.ring import Ring +from swift.common.ring.utils import is_local_device from swift.common.utils import get_logger, whataremyips, ismount, \ config_true_value, Timestamp from swift.common.daemon import Daemon @@ -58,6 +59,7 @@ class AccountReaper(Daemon): def __init__(self, conf, logger=None): self.conf = conf self.logger = logger or get_logger(conf, log_route='account-reaper') + self.bind_port = conf.get('bind_port', 6002) self.devices = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.interval = int(conf.get('interval', 3600)) @@ -159,7 +161,8 @@ class AccountReaper(Daemon): if not partition.isdigit(): continue nodes = self.get_account_ring().get_part_nodes(int(partition)) - if nodes[0]['ip'] not in self.myips or \ + if not is_local_device(self.myips, self.bind_port, + nodes[0]['ip'], nodes[0]['port']) or \ not os.path.isdir(partition_path): continue for suffix in os.listdir(partition_path): diff --git a/swift/cli/ringbuilder.py b/swift/cli/ringbuilder.py index 4a3a7d82a3..f3234bad57 100755 --- a/swift/cli/ringbuilder.py +++ b/swift/cli/ringbuilder.py @@ -16,6 +16,7 @@ from errno import EEXIST from itertools import islice, izip +from operator import itemgetter from os import mkdir from os.path import basename, abspath, dirname, exists, join as pathjoin from sys import argv as sys_argv, exit, stderr @@ -27,10 +28,12 @@ import math from swift.common import exceptions from swift.common.ring import RingBuilder, Ring from swift.common.ring.builder import MAX_BALANCE -from swift.common.utils import lock_parent_directory -from swift.common.ring.utils import parse_search_value, parse_args, \ - build_dev_from_opts, parse_builder_ring_filename_args, find_parts, \ +from swift.common.ring.utils import validate_args, \ + validate_and_normalize_ip, build_dev_from_opts, \ + parse_builder_ring_filename_args, parse_search_value, \ + parse_search_values_from_opts, parse_change_values_from_opts, \ dispersion_report +from swift.common.utils import lock_parent_directory MAJOR_VERSION = 1 MINOR_VERSION = 3 @@ -55,6 +58,106 @@ def format_device(dev): '"%(meta)s"' % copy_dev) +def _parse_search_values(argvish): + + new_cmd_format, opts, args = validate_args(argvish) + + # We'll either parse the all-in-one-string format or the + # --options format, + # but not both. If both are specified, raise an error. + try: + search_values = {} + if len(args) > 0: + if new_cmd_format or len(args) != 1: + print Commands.search.__doc__.strip() + exit(EXIT_ERROR) + search_values = parse_search_value(args[0]) + else: + search_values = parse_search_values_from_opts(opts) + return search_values + except ValueError as e: + print e + exit(EXIT_ERROR) + + +def _find_parts(devs): + devs = [d['id'] for d in devs] + if not devs or not builder._replica2part2dev: + return None + + partition_count = {} + for replica in builder._replica2part2dev: + for partition, device in enumerate(replica): + if device in devs: + if partition not in partition_count: + partition_count[partition] = 0 + partition_count[partition] += 1 + + # Sort by number of found replicas to keep the output format + sorted_partition_count = sorted( + partition_count.iteritems(), key=itemgetter(1), reverse=True) + + return sorted_partition_count + + +def _parse_list_parts_values(argvish): + + new_cmd_format, opts, args = validate_args(argvish) + + # We'll either parse the all-in-one-string format or the + # --options format, + # but not both. If both are specified, raise an error. + try: + devs = [] + if len(args) > 0: + if new_cmd_format: + print Commands.list_parts.__doc__.strip() + exit(EXIT_ERROR) + + for arg in args: + devs.extend( + builder.search_devs(parse_search_value(arg)) or []) + else: + devs.extend(builder.search_devs( + parse_search_values_from_opts(opts)) or []) + + return devs + except ValueError as e: + print e + exit(EXIT_ERROR) + + +def _parse_address(rest): + if rest.startswith('['): + # remove first [] for ip + rest = rest.replace('[', '', 1).replace(']', '', 1) + + pos = 0 + while (pos < len(rest) and + not (rest[pos] == 'R' or rest[pos] == '/')): + pos += 1 + address = rest[:pos] + rest = rest[pos:] + + port_start = address.rfind(':') + if port_start == -1: + raise ValueError('Invalid port in add value') + + ip = address[:port_start] + try: + port = int(address[(port_start + 1):]) + except (TypeError, ValueError): + raise ValueError( + 'Invalid port %s in add value' % address[port_start:]) + + # if this is an ipv6 address then we want to convert it + # to all lowercase and use its fully expanded representation + # to make searches easier + ip = validate_and_normalize_ip(ip) + + return (ip, port, rest) + + def _parse_add_values(argvish): """ Parse devices to add as specified on the command line. @@ -63,23 +166,17 @@ def _parse_add_values(argvish): :returns: array of device dicts """ + new_cmd_format, opts, args = validate_args(argvish) - opts, args = parse_args(argvish) - - # We'll either parse the all-in-one-string format or the --options format, + # We'll either parse the all-in-one-string format or the + # --options format, # but not both. If both are specified, raise an error. - opts_used = opts.region or opts.zone or opts.ip or opts.port or \ - opts.device or opts.weight or opts.meta - - if len(args) > 0 and opts_used: - print Commands.add.__doc__.strip() - exit(EXIT_ERROR) - elif len(args) > 0: - if len(args) % 2 != 0: + parsed_devs = [] + if len(args) > 0: + if new_cmd_format or len(args) % 2 != 0: print Commands.add.__doc__.strip() exit(EXIT_ERROR) - parsed_devs = [] devs_and_weights = izip(islice(args, 0, len(args), 2), islice(args, 1, len(args), 2)) @@ -93,12 +190,11 @@ def _parse_add_values(argvish): region = int(devstr[1:i]) rest = devstr[i:] else: - stderr.write("WARNING: No region specified for %s. " - "Defaulting to region 1.\n" % devstr) + stderr.write('WARNING: No region specified for %s. ' + 'Defaulting to region 1.\n' % devstr) if not rest.startswith('z'): - print 'Invalid add value: %s' % devstr - exit(EXIT_ERROR) + raise ValueError('Invalid add value: %s' % devstr) i = 1 while i < len(rest) and rest[i].isdigit(): i += 1 @@ -106,64 +202,18 @@ def _parse_add_values(argvish): rest = rest[i:] if not rest.startswith('-'): - print 'Invalid add value: %s' % devstr - print "The on-disk ring builder is unchanged.\n" - exit(EXIT_ERROR) - i = 1 - if rest[i] == '[': - i += 1 - while i < len(rest) and rest[i] != ']': - i += 1 - i += 1 - ip = rest[1:i].lstrip('[').rstrip(']') - rest = rest[i:] - else: - while i < len(rest) and rest[i] in '0123456789.': - i += 1 - ip = rest[1:i] - rest = rest[i:] + raise ValueError('Invalid add value: %s' % devstr) - if not rest.startswith(':'): - print 'Invalid add value: %s' % devstr - print "The on-disk ring builder is unchanged.\n" - exit(EXIT_ERROR) - i = 1 - while i < len(rest) and rest[i].isdigit(): - i += 1 - port = int(rest[1:i]) - rest = rest[i:] + ip, port, rest = _parse_address(rest[1:]) replication_ip = ip replication_port = port if rest.startswith('R'): - i = 1 - if rest[i] == '[': - i += 1 - while i < len(rest) and rest[i] != ']': - i += 1 - i += 1 - replication_ip = rest[1:i].lstrip('[').rstrip(']') - rest = rest[i:] - else: - while i < len(rest) and rest[i] in '0123456789.': - i += 1 - replication_ip = rest[1:i] - rest = rest[i:] - - if not rest.startswith(':'): - print 'Invalid add value: %s' % devstr - print "The on-disk ring builder is unchanged.\n" - exit(EXIT_ERROR) - i = 1 - while i < len(rest) and rest[i].isdigit(): - i += 1 - replication_port = int(rest[1:i]) - rest = rest[i:] - + replication_ip, replication_port, rest = \ + _parse_address(rest[1:]) if not rest.startswith('/'): - print 'Invalid add value: %s' % devstr - print "The on-disk ring builder is unchanged.\n" - exit(EXIT_ERROR) + raise ValueError( + 'Invalid add value: %s' % devstr) i = 1 while i < len(rest) and rest[i] != '_': i += 1 @@ -174,32 +224,227 @@ def _parse_add_values(argvish): if rest.startswith('_'): meta = rest[1:] - try: - weight = float(weightstr) - except ValueError: - print 'Invalid weight value: %s' % weightstr - print "The on-disk ring builder is unchanged.\n" - exit(EXIT_ERROR) + weight = float(weightstr) if weight < 0: - print 'Invalid weight value (must be positive): %s' % weightstr - print "The on-disk ring builder is unchanged.\n" - exit(EXIT_ERROR) + raise ValueError('Invalid weight value: %s' % devstr) parsed_devs.append({'region': region, 'zone': zone, 'ip': ip, 'port': port, 'device': device_name, 'replication_ip': replication_ip, 'replication_port': replication_port, 'weight': weight, 'meta': meta}) - return parsed_devs else: - try: - dev = build_dev_from_opts(opts) - except ValueError as e: - print e - print "The on-disk ring builder is unchanged.\n" + parsed_devs.append(build_dev_from_opts(opts)) + + return parsed_devs + + +def _set_weight_values(devs, weight): + if not devs: + print('Search value matched 0 devices.\n' + 'The on-disk ring builder is unchanged.') + exit(EXIT_ERROR) + + if len(devs) > 1: + print 'Matched more than one device:' + for dev in devs: + print ' %s' % format_device(dev) + if raw_input('Are you sure you want to update the weight for ' + 'these %s devices? (y/N) ' % len(devs)) != 'y': + print 'Aborting device modifications' exit(EXIT_ERROR) - return [dev] + + for dev in devs: + builder.set_dev_weight(dev['id'], weight) + print '%s weight set to %s' % (format_device(dev), + dev['weight']) + + +def _parse_set_weight_values(argvish): + + new_cmd_format, opts, args = validate_args(argvish) + + # We'll either parse the all-in-one-string format or the + # --options format, + # but not both. If both are specified, raise an error. + try: + devs = [] + if not new_cmd_format: + if len(args) % 2 != 0: + print Commands.set_weight.__doc__.strip() + exit(EXIT_ERROR) + + devs_and_weights = izip(islice(argvish, 0, len(argvish), 2), + islice(argvish, 1, len(argvish), 2)) + for devstr, weightstr in devs_and_weights: + devs.extend(builder.search_devs( + parse_search_value(devstr)) or []) + weight = float(weightstr) + _set_weight_values(devs, weight) + else: + if len(args) != 1: + print Commands.set_weight.__doc__.strip() + exit(EXIT_ERROR) + + devs.extend(builder.search_devs( + parse_search_values_from_opts(opts)) or []) + weight = float(args[0]) + _set_weight_values(devs, weight) + except ValueError as e: + print e + exit(EXIT_ERROR) + + +def _set_info_values(devs, change): + + if not devs: + print("Search value matched 0 devices.\n" + "The on-disk ring builder is unchanged.") + exit(EXIT_ERROR) + + if len(devs) > 1: + print 'Matched more than one device:' + for dev in devs: + print ' %s' % format_device(dev) + if raw_input('Are you sure you want to update the info for ' + 'these %s devices? (y/N) ' % len(devs)) != 'y': + print 'Aborting device modifications' + exit(EXIT_ERROR) + + for dev in devs: + orig_dev_string = format_device(dev) + test_dev = dict(dev) + for key in change: + test_dev[key] = change[key] + for check_dev in builder.devs: + if not check_dev or check_dev['id'] == test_dev['id']: + continue + if check_dev['ip'] == test_dev['ip'] and \ + check_dev['port'] == test_dev['port'] and \ + check_dev['device'] == test_dev['device']: + print 'Device %d already uses %s:%d/%s.' % \ + (check_dev['id'], check_dev['ip'], + check_dev['port'], check_dev['device']) + exit(EXIT_ERROR) + for key in change: + dev[key] = change[key] + print 'Device %s is now %s' % (orig_dev_string, + format_device(dev)) + + +def _parse_set_info_values(argvish): + + new_cmd_format, opts, args = validate_args(argvish) + + # We'll either parse the all-in-one-string format or the + # --options format, + # but not both. If both are specified, raise an error. + if not new_cmd_format: + if len(args) % 2 != 0: + print Commands.search.__doc__.strip() + exit(EXIT_ERROR) + + searches_and_changes = izip(islice(argvish, 0, len(argvish), 2), + islice(argvish, 1, len(argvish), 2)) + + for search_value, change_value in searches_and_changes: + devs = builder.search_devs(parse_search_value(search_value)) + change = {} + ip = '' + if len(change_value) and change_value[0].isdigit(): + i = 1 + while (i < len(change_value) and + change_value[i] in '0123456789.'): + i += 1 + ip = change_value[:i] + change_value = change_value[i:] + elif len(change_value) and change_value[0] == '[': + i = 1 + while i < len(change_value) and change_value[i] != ']': + i += 1 + i += 1 + ip = change_value[:i].lstrip('[').rstrip(']') + change_value = change_value[i:] + if ip: + change['ip'] = validate_and_normalize_ip(ip) + if change_value.startswith(':'): + i = 1 + while i < len(change_value) and change_value[i].isdigit(): + i += 1 + change['port'] = int(change_value[1:i]) + change_value = change_value[i:] + if change_value.startswith('R'): + change_value = change_value[1:] + replication_ip = '' + if len(change_value) and change_value[0].isdigit(): + i = 1 + while (i < len(change_value) and + change_value[i] in '0123456789.'): + i += 1 + replication_ip = change_value[:i] + change_value = change_value[i:] + elif len(change_value) and change_value[0] == '[': + i = 1 + while i < len(change_value) and change_value[i] != ']': + i += 1 + i += 1 + replication_ip = \ + change_value[:i].lstrip('[').rstrip(']') + change_value = change_value[i:] + if replication_ip: + change['replication_ip'] = \ + validate_and_normalize_ip(replication_ip) + if change_value.startswith(':'): + i = 1 + while i < len(change_value) and change_value[i].isdigit(): + i += 1 + change['replication_port'] = int(change_value[1:i]) + change_value = change_value[i:] + if change_value.startswith('/'): + i = 1 + while i < len(change_value) and change_value[i] != '_': + i += 1 + change['device'] = change_value[1:i] + change_value = change_value[i:] + if change_value.startswith('_'): + change['meta'] = change_value[1:] + change_value = '' + if change_value or not change: + raise ValueError('Invalid set info change value: %s' % + repr(argvish[1])) + _set_info_values(devs, change) + else: + devs = builder.search_devs(parse_search_values_from_opts(opts)) + change = parse_change_values_from_opts(opts) + _set_info_values(devs, change) + + +def _parse_remove_values(argvish): + + new_cmd_format, opts, args = validate_args(argvish) + + # We'll either parse the all-in-one-string format or the + # --options format, + # but not both. If both are specified, raise an error. + try: + devs = [] + if len(args) > 0: + if new_cmd_format: + print Commands.remove.__doc__.strip() + exit(EXIT_ERROR) + + for arg in args: + devs.extend(builder.search_devs( + parse_search_value(arg)) or []) + else: + devs.extend(builder.search_devs( + parse_search_values_from_opts(opts))) + + return devs + except ValueError as e: + print e + exit(EXIT_ERROR) class Commands(object): @@ -286,6 +531,18 @@ swift-ring-builder def search(): """ swift-ring-builder search + +or + +swift-ring-builder search + --region --zone --ip --port + --replication-ip --replication-port + --device --meta --weight + + Where , and are replication ip, hostname + and port. + Any of the options are optional in both cases. + Shows information about matching devices. """ if len(argv) < 4: @@ -293,7 +550,9 @@ swift-ring-builder search print print parse_search_value.__doc__.strip() exit(EXIT_ERROR) - devs = builder.search_devs(parse_search_value(argv[3])) + + devs = builder.search_devs(_parse_search_values(argv[3:])) + if not devs: print 'No matching devices found' exit(EXIT_ERROR) @@ -322,6 +581,18 @@ swift-ring-builder search def list_parts(): """ swift-ring-builder list_parts [] .. + +or + +swift-ring-builder list_parts + --region --zone --ip --port + --replication-ip --replication-port + --device --meta --weight + + Where , and are replication ip, hostname + and port. + Any of the options are optional in both cases. + Returns a 2 column list of all the partitions that are assigned to any of the devices matching the search values given. The first column is the assigned partition number and the second column is the number of device @@ -340,7 +611,12 @@ swift-ring-builder list_parts [] .. 'Please rebalance first.' % argv[1]) exit(EXIT_ERROR) - sorted_partition_count = find_parts(builder, argv) + devs = _parse_list_parts_values(argv[3:]) + if not devs: + print 'No matching devices found' + exit(EXIT_ERROR) + + sorted_partition_count = _find_parts(devs) if not sorted_partition_count: print 'No matching devices found' @@ -364,8 +640,8 @@ swift-ring-builder add or swift-ring-builder add - --region --zone --ip --port - [--replication-ip --replication-port ] + --region --zone --ip --port + [--replication-ip ] [--replication-port ] --device --weight [--meta ] @@ -373,24 +649,30 @@ swift-ring-builder add assigned to the new device until after running 'rebalance'. This is so you can make multiple device changes and rebalance them all just once. """ - if len(argv) < 5 or len(argv) % 2 != 1: + if len(argv) < 5: print Commands.add.__doc__.strip() exit(EXIT_ERROR) - for new_dev in _parse_add_values(argv[3:]): - for dev in builder.devs: - if dev is None: - continue - if dev['ip'] == new_dev['ip'] and \ - dev['port'] == new_dev['port'] and \ - dev['device'] == new_dev['device']: - print 'Device %d already uses %s:%d/%s.' % \ - (dev['id'], dev['ip'], dev['port'], dev['device']) - print "The on-disk ring builder is unchanged.\n" - exit(EXIT_ERROR) - builder.add_dev(new_dev) - print('Device %s weight %s' % - (format_device(new_dev), new_dev['weight'])) + try: + for new_dev in _parse_add_values(argv[3:]): + for dev in builder.devs: + if dev is None: + continue + if dev['ip'] == new_dev['ip'] and \ + dev['port'] == new_dev['port'] and \ + dev['device'] == new_dev['device']: + print 'Device %d already uses %s:%d/%s.' % \ + (dev['id'], dev['ip'], + dev['port'], dev['device']) + print "The on-disk ring builder is unchanged.\n" + exit(EXIT_ERROR) + dev_id = builder.add_dev(new_dev) + print('Device %s with %s weight got id %s' % + (format_device(new_dev), new_dev['weight'], dev_id)) + except ValueError as err: + print err + print 'The on-disk ring builder is unchanged.' + exit(EXIT_ERROR) builder.save(argv[1]) exit(EXIT_SUCCESS) @@ -400,38 +682,30 @@ swift-ring-builder add swift-ring-builder set_weight [ set_weight + --region --zone --ip --port + --replication-ip --replication-port + --device --meta --weight + + Where , and are replication ip, hostname + and port. + Any of the options are optional in both cases. + Resets the devices' weights. No partitions will be reassigned to or from the device until after running 'rebalance'. This is so you can make multiple device changes and rebalance them all just once. """ - if len(argv) < 5 or len(argv) % 2 != 1: + # if len(argv) < 5 or len(argv) % 2 != 1: + if len(argv) < 5: print Commands.set_weight.__doc__.strip() print print parse_search_value.__doc__.strip() exit(EXIT_ERROR) - devs_and_weights = izip(islice(argv, 3, len(argv), 2), - islice(argv, 4, len(argv), 2)) - for devstr, weightstr in devs_and_weights: - devs = builder.search_devs(parse_search_value(devstr)) - weight = float(weightstr) - if not devs: - print("Search value \"%s\" matched 0 devices.\n" - "The on-disk ring builder is unchanged.\n" - % devstr) - exit(EXIT_ERROR) - if len(devs) > 1: - print 'Matched more than one device:' - for dev in devs: - print ' %s' % format_device(dev) - if raw_input('Are you sure you want to update the weight for ' - 'these %s devices? (y/N) ' % len(devs)) != 'y': - print 'Aborting device modifications' - exit(EXIT_ERROR) - for dev in devs: - builder.set_dev_weight(dev['id'], weight) - print '%s weight set to %s' % (format_device(dev), - dev['weight']) + _parse_set_weight_values(argv[3:]) + builder.save(argv[1]) exit(EXIT_SUCCESS) @@ -441,7 +715,21 @@ swift-ring-builder set_info :[R:]/_ [ :[R:]/_] ... - Where and are replication ip and port. +or + +swift-ring-builder set_info + --ip --port + --replication-ip --replication-port + --device --meta + --change-ip --change-port + --change-replication-ip + --change-replication-port + --change-device + --change-meta + + Where , and are replication ip, hostname + and port. + Any of the options are optional in both cases. For each search-value, resets the matched device's information. This information isn't used to assign partitions, so you can use @@ -451,111 +739,36 @@ swift-ring-builder set_info want to change. For instance set_info d74 _"snet: 5.6.7.8" would just update the meta data for device id 74. """ - if len(argv) < 5 or len(argv) % 2 != 1: + if len(argv) < 5: print Commands.set_info.__doc__.strip() print print parse_search_value.__doc__.strip() exit(EXIT_ERROR) - searches_and_changes = izip(islice(argv, 3, len(argv), 2), - islice(argv, 4, len(argv), 2)) + try: + _parse_set_info_values(argv[3:]) + except ValueError as err: + print err + exit(EXIT_ERROR) - for search_value, change_value in searches_and_changes: - devs = builder.search_devs(parse_search_value(search_value)) - change = [] - if len(change_value) and change_value[0].isdigit(): - i = 1 - while (i < len(change_value) and - change_value[i] in '0123456789.'): - i += 1 - change.append(('ip', change_value[:i])) - change_value = change_value[i:] - elif len(change_value) and change_value[0] == '[': - i = 1 - while i < len(change_value) and change_value[i] != ']': - i += 1 - i += 1 - change.append(('ip', change_value[:i].lstrip('[').rstrip(']'))) - change_value = change_value[i:] - if change_value.startswith(':'): - i = 1 - while i < len(change_value) and change_value[i].isdigit(): - i += 1 - change.append(('port', int(change_value[1:i]))) - change_value = change_value[i:] - if change_value.startswith('R'): - change_value = change_value[1:] - if len(change_value) and change_value[0].isdigit(): - i = 1 - while (i < len(change_value) and - change_value[i] in '0123456789.'): - i += 1 - change.append(('replication_ip', change_value[:i])) - change_value = change_value[i:] - elif len(change_value) and change_value[0] == '[': - i = 1 - while i < len(change_value) and change_value[i] != ']': - i += 1 - i += 1 - change.append(('replication_ip', - change_value[:i].lstrip('[').rstrip(']'))) - change_value = change_value[i:] - if change_value.startswith(':'): - i = 1 - while i < len(change_value) and change_value[i].isdigit(): - i += 1 - change.append(('replication_port', int(change_value[1:i]))) - change_value = change_value[i:] - if change_value.startswith('/'): - i = 1 - while i < len(change_value) and change_value[i] != '_': - i += 1 - change.append(('device', change_value[1:i])) - change_value = change_value[i:] - if change_value.startswith('_'): - change.append(('meta', change_value[1:])) - change_value = '' - if change_value or not change: - raise ValueError('Invalid set info change value: %s' % - repr(argv[4])) - if not devs: - print("Search value \"%s\" matched 0 devices.\n" - "The on-disk ring builder is unchanged.\n" - % search_value) - exit(EXIT_ERROR) - if len(devs) > 1: - print 'Matched more than one device:' - for dev in devs: - print ' %s' % format_device(dev) - if raw_input('Are you sure you want to update the info for ' - 'these %s devices? (y/N) ' % len(devs)) != 'y': - print 'Aborting device modifications' - exit(EXIT_ERROR) - for dev in devs: - orig_dev_string = format_device(dev) - test_dev = dict(dev) - for key, value in change: - test_dev[key] = value - for check_dev in builder.devs: - if not check_dev or check_dev['id'] == test_dev['id']: - continue - if check_dev['ip'] == test_dev['ip'] and \ - check_dev['port'] == test_dev['port'] and \ - check_dev['device'] == test_dev['device']: - print 'Device %d already uses %s:%d/%s.' % \ - (check_dev['id'], check_dev['ip'], - check_dev['port'], check_dev['device']) - exit(EXIT_ERROR) - for key, value in change: - dev[key] = value - print 'Device %s is now %s' % (orig_dev_string, - format_device(dev)) builder.save(argv[1]) exit(EXIT_SUCCESS) def remove(): """ swift-ring-builder remove [search-value ...] + +or + +swift-ring-builder search + --region --zone --ip --port + --replication-ip --replication-port + --device --meta --weight + + Where , and are replication ip, hostname + and port. + Any of the options are optional in both cases. + Removes the device(s) from the ring. This should normally just be used for a device that has failed. For a device you wish to decommission, it's best to set its weight to 0, wait for it to drain all its data, then use this @@ -569,36 +782,37 @@ swift-ring-builder remove [search-value ...] print parse_search_value.__doc__.strip() exit(EXIT_ERROR) - for search_value in argv[3:]: - devs = builder.search_devs(parse_search_value(search_value)) - if not devs: - print("Search value \"%s\" matched 0 devices.\n" - "The on-disk ring builder is unchanged." % search_value) - exit(EXIT_ERROR) - if len(devs) > 1: - print 'Matched more than one device:' - for dev in devs: - print ' %s' % format_device(dev) - if raw_input('Are you sure you want to remove these %s ' - 'devices? (y/N) ' % len(devs)) != 'y': - print 'Aborting device removals' - exit(EXIT_ERROR) + devs = _parse_remove_values(argv[3:]) + + if not devs: + print('Search value matched 0 devices.\n' + 'The on-disk ring builder is unchanged.') + exit(EXIT_ERROR) + + if len(devs) > 1: + print 'Matched more than one device:' for dev in devs: - try: - builder.remove_dev(dev['id']) - except exceptions.RingBuilderError as e: - print '-' * 79 - print( - "An error occurred while removing device with id %d\n" - "This usually means that you attempted to remove\n" - "the last device in a ring. If this is the case,\n" - "consider creating a new ring instead.\n" - "The on-disk ring builder is unchanged.\n" - "Original exception message: %s" % - (dev['id'], e) - ) - print '-' * 79 - exit(EXIT_ERROR) + print ' %s' % format_device(dev) + if raw_input('Are you sure you want to remove these %s ' + 'devices? (y/N) ' % len(devs)) != 'y': + print 'Aborting device removals' + exit(EXIT_ERROR) + + for dev in devs: + try: + builder.remove_dev(dev['id']) + except exceptions.RingBuilderError as e: + print '-' * 79 + print( + 'An error occurred while removing device with id %d\n' + 'This usually means that you attempted to remove\n' + 'the last device in a ring. If this is the case,\n' + 'consider creating a new ring instead.\n' + 'The on-disk ring builder is unchanged.\n' + 'Original exception message: %s' % + (dev['id'], e)) + print '-' * 79 + exit(EXIT_ERROR) print '%s marked for removal and will ' \ 'be removed next rebalance.' % format_device(dev) diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index e22e9711ef..e456beed75 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -33,6 +33,7 @@ from swift.common.utils import get_logger, whataremyips, storage_directory, \ renamer, mkdirs, lock_parent_directory, config_true_value, \ unlink_older_than, dump_recon_cache, rsync_ip, ismount, json, Timestamp from swift.common import ring +from swift.common.ring.utils import is_local_device from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE from swift.common.bufferedhttp import BufferedHTTPConnection from swift.common.exceptions import DriveNotMounted @@ -543,8 +544,9 @@ class Replicator(Daemon): return self._local_device_ids = set() for node in self.ring.devs: - if (node and node['replication_ip'] in ips and - node['replication_port'] == self.port): + if node and is_local_device(ips, self.port, + node['replication_ip'], + node['replication_port']): if self.mount_check and not ismount( os.path.join(self.root, node['device'])): self.logger.warn( diff --git a/swift/common/ring/builder.py b/swift/common/ring/builder.py index 9bebab27e8..eacda5ffba 100644 --- a/swift/common/ring/builder.py +++ b/swift/common/ring/builder.py @@ -27,7 +27,8 @@ from time import time from swift.common import exceptions from swift.common.ring import RingData -from swift.common.ring.utils import tiers_for_dev, build_tier_tree +from swift.common.ring.utils import tiers_for_dev, build_tier_tree, \ + validate_and_normalize_address MAX_BALANCE = 999.99 @@ -1305,6 +1306,15 @@ class RingBuilder(object): if key == 'meta': if value not in dev.get(key): matched = False + elif key == 'ip' or key == 'replication_ip': + cdev = '' + try: + cdev = validate_and_normalize_address( + dev.get(key, '')) + except ValueError: + pass + if cdev != value: + matched = False elif dev.get(key) != value: matched = False if matched: diff --git a/swift/common/ring/utils.py b/swift/common/ring/utils.py index 2ca2134605..17186dee87 100644 --- a/swift/common/ring/utils.py +++ b/swift/common/ring/utils.py @@ -13,9 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. from collections import defaultdict -from operator import itemgetter import optparse import re +import socket + +from swift.common.utils import expand_ipv6 def tiers_for_dev(dev): @@ -128,10 +130,136 @@ def build_tier_tree(devices): return tier2children +def validate_and_normalize_ip(ip): + """ + Return normalized ip if the ip is a valid ip. + Otherwise raise ValueError Exception. The hostname is + normalized to all lower case. IPv6-addresses are converted to + lowercase and fully expanded. + """ + # first convert to lower case + new_ip = ip.lower() + if is_valid_ipv4(new_ip): + return new_ip + elif is_valid_ipv6(new_ip): + return expand_ipv6(new_ip) + else: + raise ValueError('Invalid ip %s' % ip) + + +def validate_and_normalize_address(address): + """ + Return normalized address if the address is a valid ip or hostname. + Otherwise raise ValueError Exception. The hostname is + normalized to all lower case. IPv6-addresses are converted to + lowercase and fully expanded. + + RFC1123 2.1 Host Names and Nubmers + DISCUSSION + This last requirement is not intended to specify the complete + syntactic form for entering a dotted-decimal host number; + that is considered to be a user-interface issue. For + example, a dotted-decimal number must be enclosed within + "[ ]" brackets for SMTP mail (see Section 5.2.17). This + notation could be made universal within a host system, + simplifying the syntactic checking for a dotted-decimal + number. + + If a dotted-decimal number can be entered without such + identifying delimiters, then a full syntactic check must be + made, because a segment of a host domain name is now allowed + to begin with a digit and could legally be entirely numeric + (see Section 6.1.2.4). However, a valid host name can never + have the dotted-decimal form #.#.#.#, since at least the + highest-level component label will be alphabetic. + """ + new_address = address.lstrip('[').rstrip(']') + if address.startswith('[') and address.endswith(']'): + return validate_and_normalize_ip(new_address) + + new_address = new_address.lower() + if is_valid_ipv4(new_address): + return new_address + elif is_valid_ipv6(new_address): + return expand_ipv6(new_address) + elif is_valid_hostname(new_address): + return new_address + else: + raise ValueError('Invalid address %s' % address) + + +def is_valid_ip(ip): + """ + Return True if the provided ip is a valid IP-address + """ + return is_valid_ipv4(ip) or is_valid_ipv6(ip) + + +def is_valid_ipv4(ip): + """ + Return True if the provided ip is a valid IPv4-address + """ + try: + socket.inet_pton(socket.AF_INET, ip) + except socket.error: + return False + return True + + +def is_valid_ipv6(ip): + """ + Return True if the provided ip is a valid IPv6-address + """ + try: + socket.inet_pton(socket.AF_INET6, ip) + except socket.error: # not a valid address + return False + return True + + +def is_valid_hostname(hostname): + """ + Return True if the provided hostname is a valid hostname + """ + if len(hostname) < 1 or len(hostname) > 255: + return False + if hostname[-1] == ".": + # strip exactly one dot from the right, if present + hostname = hostname[:-1] + allowed = re.compile("(?!-)[A-Z\d-]{1,63}(? can be of the form:: - drz-:[R:]/ + drz-:R:/ _ Where and are replication ip and port. @@ -201,6 +329,12 @@ def parse_search_value(search_value): i += 1 match['ip'] = search_value[:i].lstrip('[').rstrip(']') search_value = search_value[i:] + + if 'ip' in match: + # ipv6 addresses are converted to all lowercase + # and use the fully expanded representation + match['ip'] = validate_and_normalize_ip(match['ip']) + if search_value.startswith(':'): i = 1 while i < len(search_value) and search_value[i].isdigit(): @@ -224,6 +358,13 @@ def parse_search_value(search_value): i += 1 match['replication_ip'] = search_value[:i].lstrip('[').rstrip(']') search_value = search_value[i:] + + if 'replication_ip' in match: + # ipv6 addresses are converted to all lowercase + # and use the fully expanded representation + match['replication_ip'] = \ + validate_and_normalize_ip(match['replication_ip']) + if search_value.startswith(':'): i = 1 while i < len(search_value) and search_value[i].isdigit(): @@ -245,11 +386,68 @@ def parse_search_value(search_value): return match +def parse_search_values_from_opts(opts): + """ + Convert optparse style options into a dictionary for searching. + + :param opts: optparse style options + :returns: a dictonary with search values to filter devices, + supported parameters are id, region, zone, ip, port, + replication_ip, replication_port, device, weight, meta + """ + + search_values = {} + for key in ('id', 'region', 'zone', 'ip', 'port', 'replication_ip', + 'replication_port', 'device', 'weight', 'meta'): + value = getattr(opts, key, None) + if value: + if key == 'ip' or key == 'replication_ip': + value = validate_and_normalize_address(value) + search_values[key] = value + return search_values + + +def parse_change_values_from_opts(opts): + """ + Convert optparse style options into a dictionary for changing. + + :param opts: optparse style options + :returns: a dictonary with change values to filter devices, + supported parameters are ip, port, replication_ip, + replication_port + """ + + change_values = {} + for key in ('change_ip', 'change_port', 'change_replication_ip', + 'change_replication_port', 'change_device', 'change_meta'): + value = getattr(opts, key, None) + if value: + if key == 'change_ip' or key == 'change_replication_ip': + value = validate_and_normalize_address(value) + change_values[key.replace('change_', '')] = value + return change_values + + +def validate_args(argvish): + """ + Build OptionParse and validate it whether the format is new command-line + format or not. + """ + opts, args = parse_args(argvish) + new_cmd_format = opts.id or opts.region or opts.zone or \ + opts.ip or opts.port or \ + opts.replication_ip or opts.replication_port or \ + opts.device or opts.weight or opts.meta + return (new_cmd_format, opts, args) + + def parse_args(argvish): """ Build OptionParser and evaluate command line arguments. """ parser = optparse.OptionParser() + parser.add_option('-u', '--id', type="int", + help="Device ID") parser.add_option('-r', '--region', type="int", help="Region") parser.add_option('-z', '--zone', type="int", @@ -268,6 +466,18 @@ def parse_args(argvish): help="Device weight") parser.add_option('-m', '--meta', type="string", default="", help="Extra device info (just a string)") + parser.add_option('-I', '--change-ip', type="string", + help="IP address for change") + parser.add_option('-P', '--change-port', type="int", + help="Port number for change") + parser.add_option('-J', '--change-replication-ip', type="string", + help="Replication IP address for change") + parser.add_option('-Q', '--change-replication-port', type="int", + help="Replication port number for change") + parser.add_option('-D', '--change-device', type="string", + help="Device name (e.g. md0, sdb1) for change") + parser.add_option('-M', '--change-meta', type="string", default="", + help="Extra device info (just a string) for change") return parser.parse_args(argvish) @@ -300,40 +510,17 @@ def build_dev_from_opts(opts): raise ValueError('Required argument %s/%s not specified.' % (shortopt, longopt)) - replication_ip = opts.replication_ip or opts.ip + ip = validate_and_normalize_address(opts.ip) + replication_ip = validate_and_normalize_address( + (opts.replication_ip or opts.ip)) replication_port = opts.replication_port or opts.port - return {'region': opts.region, 'zone': opts.zone, 'ip': opts.ip, + return {'region': opts.region, 'zone': opts.zone, 'ip': ip, 'port': opts.port, 'device': opts.device, 'meta': opts.meta, 'replication_ip': replication_ip, 'replication_port': replication_port, 'weight': opts.weight} -def find_parts(builder, argv): - devs = [] - for arg in argv[3:]: - devs.extend(builder.search_devs(parse_search_value(arg)) or []) - - devs = [d['id'] for d in devs] - - if not devs: - return None - - partition_count = {} - for replica in builder._replica2part2dev: - for partition, device in enumerate(replica): - if device in devs: - if partition not in partition_count: - partition_count[partition] = 0 - partition_count[partition] += 1 - - # Sort by number of found replicas to keep the output format - sorted_partition_count = sorted( - partition_count.iteritems(), key=itemgetter(1), reverse=True) - - return sorted_partition_count - - def dispersion_report(builder, search_filter=None, verbose=False): if not builder._dispersion_graph: builder._build_dispersion_graph() diff --git a/swift/common/utils.py b/swift/common/utils.py index f04860a341..752a64bf4c 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -907,7 +907,7 @@ class NullLogger(object): """A no-op logger for eventlet wsgi.""" def write(self, *args): - #"Logs" the args to nowhere + # "Logs" the args to nowhere pass @@ -1069,6 +1069,7 @@ class LoggingHandlerWeakRef(weakref.ref): Like a weak reference, but passes through a couple methods that logging handlers need. """ + def close(self): referent = self() try: @@ -1542,6 +1543,17 @@ def parse_options(parser=None, once=False, test_args=None): return config, options +def expand_ipv6(address): + """ + Expand ipv6 address. + :param address: a string indicating valid ipv6 address + :returns: a string indicating fully expanded ipv6 address + + """ + packed_ip = socket.inet_pton(socket.AF_INET6, address) + return socket.inet_ntop(socket.AF_INET6, packed_ip) + + def whataremyips(): """ Get the machine's ip addresses @@ -1561,7 +1573,7 @@ def whataremyips(): # If we have an ipv6 address remove the # %ether_interface at the end if family == netifaces.AF_INET6: - addr = addr.split('%')[0] + addr = expand_ipv6(addr.split('%')[0]) addresses.append(addr) except ValueError: pass @@ -2388,7 +2400,7 @@ def dump_recon_cache(cache_dict, cache_file, logger, lock_timeout=2): if existing_entry: cache_entry = json.loads(existing_entry) except ValueError: - #file doesn't have a valid entry, we'll recreate it + # file doesn't have a valid entry, we'll recreate it pass for cache_key, cache_value in cache_dict.items(): put_recon_cache_entry(cache_entry, cache_key, cache_value) @@ -2724,14 +2736,15 @@ def tpool_reraise(func, *args, **kwargs): class ThreadPool(object): - BYTE = 'a'.encode('utf-8') - """ Perform blocking operations in background threads. Call its methods from within greenlets to green-wait for results without blocking the eventlet reactor (hopefully). """ + + BYTE = 'a'.encode('utf-8') + def __init__(self, nthreads=2): self.nthreads = nthreads self._run_queue = Queue() diff --git a/swift/container/sync.py b/swift/container/sync.py index 27cd0d04f3..4bf5fc5c36 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -29,6 +29,7 @@ from swift.common.direct_client import direct_get_object from swift.common.internal_client import delete_object, put_object from swift.common.exceptions import ClientException from swift.common.ring import Ring +from swift.common.ring.utils import is_local_device from swift.common.utils import ( audit_location_generator, clean_content_type, config_true_value, FileLikeIter, get_logger, hash_path, quote, urlparse, validate_sync_to, @@ -239,7 +240,8 @@ class ContainerSync(Daemon): x, nodes = self.container_ring.get_nodes(info['account'], info['container']) for ordinal, node in enumerate(nodes): - if node['ip'] in self._myips and node['port'] == self._myport: + if is_local_device(self._myips, self._myport, + node['ip'], node['port']): break else: return diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index 5cad5d4cf0..794fc44827 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -27,6 +27,7 @@ from eventlet import GreenPool, tpool, Timeout, sleep, hubs from eventlet.green import subprocess from eventlet.support.greenlets import GreenletExit +from swift.common.ring.utils import is_local_device from swift.common.utils import whataremyips, unlink_older_than, \ compute_eta, get_logger, dump_recon_cache, ismount, \ rsync_ip, mkdirs, config_true_value, list_from_csv, get_hub, \ @@ -417,8 +418,10 @@ class ObjectReplicator(Daemon): data_dir = get_data_dir(policy.idx) for local_dev in [dev for dev in obj_ring.devs if (dev - and dev['replication_ip'] in ips - and dev['replication_port'] == self.port + and is_local_device(ips, + self.port, + dev['replication_ip'], + dev['replication_port']) and (override_devices is None or dev['device'] in override_devices))]: dev_path = join(self.devices_dir, local_dev['device']) diff --git a/test/unit/account/test_reaper.py b/test/unit/account/test_reaper.py index deec2e3301..f1ccb4b242 100644 --- a/test/unit/account/test_reaper.py +++ b/test/unit/account/test_reaper.py @@ -95,15 +95,15 @@ class FakeRing(object): def __init__(self): self.nodes = [{'id': '1', 'ip': '10.10.10.1', - 'port': None, + 'port': 6002, 'device': None}, {'id': '2', 'ip': '10.10.10.1', - 'port': None, + 'port': 6002, 'device': None}, {'id': '3', 'ip': '10.10.10.1', - 'port': None, + 'port': 6002, 'device': None}, ] diff --git a/test/unit/cli/test_ringbuilder.py b/test/unit/cli/test_ringbuilder.py index b0a1e1dfcb..9361c78d89 100644 --- a/test/unit/cli/test_ringbuilder.py +++ b/test/unit/cli/test_ringbuilder.py @@ -20,7 +20,7 @@ import tempfile import unittest import uuid -import swift.cli.ringbuilder +from swift.cli import ringbuilder from swift.common import exceptions from swift.common.ring import RingBuilder @@ -36,7 +36,7 @@ class RunSwiftRingBuilderMixin(object): try: with mock.patch("sys.stdout", mock_stdout): with mock.patch("sys.stderr", mock_stderr): - swift.cli.ringbuilder.main(srb_args) + ringbuilder.main(srb_args) except SystemExit as err: if err.code not in (0, 1): # (success, warning) raise @@ -96,30 +96,177 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin): }) ring.save(self.tmpfile) + def test_parse_search_values_old_format(self): + # Test old format + argv = ["d0r0z0-127.0.0.1:6000R127.0.0.1:6000/sda1_some meta data"] + search_values = ringbuilder._parse_search_values(argv) + self.assertEqual(search_values['id'], 0) + self.assertEqual(search_values['region'], 0) + self.assertEqual(search_values['zone'], 0) + self.assertEqual(search_values['ip'], '127.0.0.1') + self.assertEqual(search_values['port'], 6000) + self.assertEqual(search_values['replication_ip'], '127.0.0.1') + self.assertEqual(search_values['replication_port'], 6000) + self.assertEqual(search_values['device'], 'sda1') + self.assertEqual(search_values['meta'], 'some meta data') + + def test_parse_search_values_new_format(self): + # Test new format + argv = ["--id", "0", "--region", "0", "--zone", "0", + "--ip", "127.0.0.1", + "--port", "6000", + "--replication-ip", "127.0.0.1", + "--replication-port", "6000", + "--device", "sda1", "--meta", "some meta data", + "--weight", "100"] + search_values = ringbuilder._parse_search_values(argv) + self.assertEqual(search_values['id'], 0) + self.assertEqual(search_values['region'], 0) + self.assertEqual(search_values['zone'], 0) + self.assertEqual(search_values['ip'], '127.0.0.1') + self.assertEqual(search_values['port'], 6000) + self.assertEqual(search_values['replication_ip'], '127.0.0.1') + self.assertEqual(search_values['replication_port'], 6000) + self.assertEqual(search_values['device'], 'sda1') + self.assertEqual(search_values['meta'], 'some meta data') + self.assertEqual(search_values['weight'], 100) + + def test_parse_search_values_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["--region", "2", "test"] + err = None + try: + ringbuilder._parse_search_values(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_find_parts(self): + rb = RingBuilder(8, 3, 0) + rb.add_dev({'id': 0, 'region': 1, 'zone': 0, 'weight': 100, + 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) + rb.add_dev({'id': 1, 'region': 1, 'zone': 1, 'weight': 100, + 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) + rb.add_dev({'id': 2, 'region': 1, 'zone': 2, 'weight': 100, + 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) + rb.rebalance() + + rb.add_dev({'id': 3, 'region': 2, 'zone': 1, 'weight': 10, + 'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'}) + rb.pretend_min_part_hours_passed() + rb.rebalance() + + ringbuilder.builder = rb + sorted_partition_count = ringbuilder._find_parts( + rb.search_devs({'ip': '127.0.0.1'})) + + # Expect 256 partitions in the output + self.assertEqual(256, len(sorted_partition_count)) + + # Each partitions should have 3 replicas + for partition, count in sorted_partition_count: + self.assertEqual( + 3, count, "Partition %d has only %d replicas" % + (partition, count)) + + def test_parse_list_parts_values_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["--region", "2", "test"] + err = None + try: + ringbuilder._parse_list_parts_values(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_parse_address_old_format(self): + # Test old format + argv = "127.0.0.1:6000R127.0.0.1:6000/sda1_some meta data" + ip, port, rest = ringbuilder._parse_address(argv) + self.assertEqual(ip, '127.0.0.1') + self.assertEqual(port, 6000) + self.assertEqual(rest, 'R127.0.0.1:6000/sda1_some meta data') + + def test_parse_add_values_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["--region", "2", "test"] + err = None + try: + ringbuilder._parse_add_values(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_set_weight_values_no_devices(self): + # Test no devices + err = None + try: + ringbuilder._set_weight_values([], 100) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_parse_set_weight_values_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["r1", "100", "r2"] + err = None + try: + ringbuilder._parse_set_weight_values(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + argv = ["--region", "2"] + err = None + try: + ringbuilder._parse_set_weight_values(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_set_info_values_no_devices(self): + # Test no devices + err = None + try: + ringbuilder._set_info_values([], 100) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_parse_set_info_values_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["r1", "127.0.0.1", "r2"] + err = None + try: + ringbuilder._parse_set_info_values(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_parse_remove_values_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["--region", "2", "test"] + err = None + try: + ringbuilder._parse_remove_values(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + def test_create_ring(self): argv = ["", self.tmpfile, "create", "6", "3.14159265359", "1"] - self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) + self.assertRaises(SystemExit, ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.part_power, 6) self.assertEqual(ring.replicas, 3.14159265359) self.assertEqual(ring.min_part_hours, 1) - def test_list_parts(self): - self.create_sample_ring() - argv = ["", self.tmpfile, "list_parts", "r1"] - - err = None - try: - swift.cli.ringbuilder.main(argv) - except SystemExit as e: - err = e - self.assertEqual(err.code, 2) - - def test_add_device(self): + def test_add_device_ipv4_old_format(self): self.create_sample_ring() + # Test ipv4(old format) argv = ["", self.tmpfile, "add", "r2z3-127.0.0.1:6000/sda3_some meta data", "3.14159265359"] - self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) + self.assertRaises(SystemExit, ringbuilder.main, argv) # Check that device was created with given data ring = RingBuilder.load(self.tmpfile) @@ -134,15 +281,148 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin): self.assertEqual(dev['replication_port'], 6000) self.assertEqual(dev['meta'], 'some meta data') + def test_add_device_ipv6_old_format(self): + self.create_sample_ring() + # Test ipv6(old format) + argv = \ + ["", self.tmpfile, "add", + "r2z3-2001:0000:1234:0000:0000:C1C0:ABCD:0876:6000" + "R2::10:7000/sda3_some meta data", + "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Check that device was created with given data + ring = RingBuilder.load(self.tmpfile) + dev = [d for d in ring.devs if d['id'] == 2][0] + self.assertEqual(dev['region'], 2) + self.assertEqual(dev['zone'], 3) + self.assertEqual(dev['ip'], '2001:0:1234::c1c0:abcd:876') + self.assertEqual(dev['port'], 6000) + self.assertEqual(dev['device'], 'sda3') + self.assertEqual(dev['weight'], 3.14159265359) + self.assertEqual(dev['replication_ip'], '2::10') + self.assertEqual(dev['replication_port'], 7000) + self.assertEqual(dev['meta'], 'some meta data') # Final check, rebalance and check ring is ok ring.rebalance() self.assertTrue(ring.validate()) + def test_add_device_ipv4_new_format(self): + self.create_sample_ring() + # Test ipv4(new format) + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "127.0.0.2", + "--port", "6000", + "--replication-ip", "127.0.0.2", + "--replication-port", "6000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Check that device was created with given data + ring = RingBuilder.load(self.tmpfile) + dev = [d for d in ring.devs if d['id'] == 2][0] + self.assertEqual(dev['region'], 2) + self.assertEqual(dev['zone'], 3) + self.assertEqual(dev['ip'], '127.0.0.2') + self.assertEqual(dev['port'], 6000) + self.assertEqual(dev['device'], 'sda3') + self.assertEqual(dev['weight'], 3.14159265359) + self.assertEqual(dev['replication_ip'], '127.0.0.2') + self.assertEqual(dev['replication_port'], 6000) + self.assertEqual(dev['meta'], 'some meta data') + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_add_device_ipv6_new_format(self): + self.create_sample_ring() + # Test ipv6(new format) + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "[3001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[3::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Check that device was created with given data + ring = RingBuilder.load(self.tmpfile) + dev = [d for d in ring.devs if d['id'] == 2][0] + self.assertEqual(dev['region'], 2) + self.assertEqual(dev['zone'], 3) + self.assertEqual(dev['ip'], '3001:0:1234::c1c0:abcd:876') + self.assertEqual(dev['port'], 6000) + self.assertEqual(dev['device'], 'sda3') + self.assertEqual(dev['weight'], 3.14159265359) + self.assertEqual(dev['replication_ip'], '3::10') + self.assertEqual(dev['replication_port'], 7000) + self.assertEqual(dev['meta'], 'some meta data') + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_add_device_domain_new_format(self): + self.create_sample_ring() + # Test domain name + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Check that device was created with given data + ring = RingBuilder.load(self.tmpfile) + dev = [d for d in ring.devs if d['id'] == 2][0] + self.assertEqual(dev['region'], 2) + self.assertEqual(dev['zone'], 3) + self.assertEqual(dev['ip'], 'test.test.com') + self.assertEqual(dev['port'], 6000) + self.assertEqual(dev['device'], 'sda3') + self.assertEqual(dev['weight'], 3.14159265359) + self.assertEqual(dev['replication_ip'], 'r.test.com') + self.assertEqual(dev['replication_port'], 7000) + self.assertEqual(dev['meta'], 'some meta data') + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_add_device_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["", self.tmpfile, "add"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_add_device_already_exists(self): + # Test Add a device that already exists + argv = ["", self.tmpfile, "add", + "r0z0-127.0.0.1:6000/sda1_some meta data", "100"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + def test_remove_device(self): for search_value in self.search_values: self.create_sample_ring() argv = ["", self.tmpfile, "remove", search_value] - self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) + self.assertRaises(SystemExit, ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) # Check that weight was set to 0 @@ -170,13 +450,269 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin): ring.rebalance() self.assertTrue(ring.validate()) + def test_remove_device_ipv4_old_format(self): + self.create_sample_ring() + # Test ipv4(old format) + argv = ["", self.tmpfile, "remove", + "d0r0z0-127.0.0.1:6000R127.0.0.1:6000/sda1_some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + + # Check that weight was set to 0 + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['weight'], 0) + + # Check that device is in list of devices to be removed + dev = [d for d in ring._remove_devs if d['id'] == 0][0] + self.assertEqual(dev['region'], 0) + self.assertEqual(dev['zone'], 0) + self.assertEqual(dev['ip'], '127.0.0.1') + self.assertEqual(dev['port'], 6000) + self.assertEqual(dev['device'], 'sda1') + self.assertEqual(dev['weight'], 0) + self.assertEqual(dev['replication_ip'], '127.0.0.1') + self.assertEqual(dev['replication_port'], 6000) + self.assertEqual(dev['meta'], 'some meta data') + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['weight'], 100) + self.assertFalse([d for d in ring._remove_devs if d['id'] == 1]) + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_remove_device_ipv6_old_format(self): + self.create_sample_ring() + # add IPV6 + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "[2001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[2::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test ipv6(old format) + argv = ["", self.tmpfile, "remove", + "d2r2z3-[2001:0000:1234:0000:0000:C1C0:ABCD:0876]:6000" + "R[2::10]:7000/sda3_some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['weight'], 100) + self.assertFalse([d for d in ring._remove_devs if d['id'] == 0]) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['weight'], 100) + self.assertFalse([d for d in ring._remove_devs if d['id'] == 1]) + + # Check that weight was set to 0 + dev = [d for d in ring.devs if d['id'] == 2][0] + self.assertEqual(dev['weight'], 0) + + # Check that device is in list of devices to be removed + dev = [d for d in ring._remove_devs if d['id'] == 2][0] + self.assertEqual(dev['region'], 2) + self.assertEqual(dev['zone'], 3) + self.assertEqual(dev['ip'], '2001:0:1234::c1c0:abcd:876') + self.assertEqual(dev['port'], 6000) + self.assertEqual(dev['device'], 'sda3') + self.assertEqual(dev['weight'], 0) + self.assertEqual(dev['replication_ip'], '2::10') + self.assertEqual(dev['replication_port'], 7000) + self.assertEqual(dev['meta'], 'some meta data') + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_remove_device_ipv4_new_format(self): + self.create_sample_ring() + # Test ipv4(new format) + argv = \ + ["", self.tmpfile, "remove", + "--id", "0", "--region", "0", "--zone", "0", + "--ip", "127.0.0.1", + "--port", "6000", + "--replication-ip", "127.0.0.1", + "--replication-port", "6000", + "--device", "sda1", "--meta", "some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + + # Check that weight was set to 0 + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['weight'], 0) + + # Check that device is in list of devices to be removed + dev = [d for d in ring._remove_devs if d['id'] == 0][0] + self.assertEqual(dev['region'], 0) + self.assertEqual(dev['zone'], 0) + self.assertEqual(dev['ip'], '127.0.0.1') + self.assertEqual(dev['port'], 6000) + self.assertEqual(dev['device'], 'sda1') + self.assertEqual(dev['weight'], 0) + self.assertEqual(dev['replication_ip'], '127.0.0.1') + self.assertEqual(dev['replication_port'], 6000) + self.assertEqual(dev['meta'], 'some meta data') + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['weight'], 100) + self.assertFalse([d for d in ring._remove_devs if d['id'] == 1]) + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_remove_device_ipv6_new_format(self): + self.create_sample_ring() + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "[3001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "8000", + "--replication-ip", "[3::10]", + "--replication-port", "9000", + "--device", "sda30", "--meta", "other meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test ipv6(new format) + argv = \ + ["", self.tmpfile, "remove", + "--id", "2", "--region", "2", "--zone", "3", + "--ip", "[3001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "8000", + "--replication-ip", "[3::10]", + "--replication-port", "9000", + "--device", "sda30", "--meta", "other meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['weight'], 100) + self.assertFalse([d for d in ring._remove_devs if d['id'] == 0]) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['weight'], 100) + self.assertFalse([d for d in ring._remove_devs if d['id'] == 1]) + + # Check that weight was set to 0 + dev = [d for d in ring.devs if d['id'] == 2][0] + self.assertEqual(dev['weight'], 0) + + # Check that device is in list of devices to be removed + dev = [d for d in ring._remove_devs if d['id'] == 2][0] + self.assertEqual(dev['region'], 2) + self.assertEqual(dev['zone'], 3) + self.assertEqual(dev['ip'], '3001:0:1234::c1c0:abcd:876') + self.assertEqual(dev['port'], 8000) + self.assertEqual(dev['device'], 'sda30') + self.assertEqual(dev['weight'], 0) + self.assertEqual(dev['replication_ip'], '3::10') + self.assertEqual(dev['replication_port'], 9000) + self.assertEqual(dev['meta'], 'other meta data') + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_remove_device_domain_new_format(self): + self.create_sample_ring() + # add domain name + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test domain name + argv = \ + ["", self.tmpfile, "remove", + "--id", "2", "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['weight'], 100) + self.assertFalse([d for d in ring._remove_devs if d['id'] == 0]) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['weight'], 100) + self.assertFalse([d for d in ring._remove_devs if d['id'] == 1]) + + # Check that weight was set to 0 + dev = [d for d in ring.devs if d['id'] == 2][0] + self.assertEqual(dev['weight'], 0) + + # Check that device is in list of devices to be removed + dev = [d for d in ring._remove_devs if d['id'] == 2][0] + self.assertEqual(dev['region'], 2) + self.assertEqual(dev['zone'], 3) + self.assertEqual(dev['ip'], 'test.test.com') + self.assertEqual(dev['port'], 6000) + self.assertEqual(dev['device'], 'sda3') + self.assertEqual(dev['weight'], 0) + self.assertEqual(dev['replication_ip'], 'r.test.com') + self.assertEqual(dev['replication_port'], 7000) + self.assertEqual(dev['meta'], 'some meta data') + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_remove_device_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["", self.tmpfile, "remove"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_remove_device_no_matching(self): + self.create_sample_ring() + # Test No matching devices + argv = ["", self.tmpfile, "remove", + "--ip", "unknown"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + def test_set_weight(self): for search_value in self.search_values: self.create_sample_ring() argv = ["", self.tmpfile, "set_weight", search_value, "3.14159265359"] - self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) + self.assertRaises(SystemExit, ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) # Check that weight was changed @@ -191,13 +727,203 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin): ring.rebalance() self.assertTrue(ring.validate()) + def test_set_weight_ipv4_old_format(self): + self.create_sample_ring() + # Test ipv4(old format) + argv = ["", self.tmpfile, "set_weight", + "d0r0z0-127.0.0.1:6000R127.0.0.1:6000/sda1_some meta data", + "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + + # Check that weight was changed + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['weight'], 3.14159265359) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['weight'], 100) + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_set_weight_ipv6_old_format(self): + self.create_sample_ring() + # add IPV6 + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "[2001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[2::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "100"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test ipv6(old format) + argv = ["", self.tmpfile, "set_weight", + "d2r2z3-[2001:0000:1234:0000:0000:C1C0:ABCD:0876]:6000" + "R[2::10]:7000/sda3_some meta data", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['weight'], 100) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['weight'], 100) + + # Check that weight was changed + dev = [d for d in ring.devs if d['id'] == 2][0] + self.assertEqual(dev['weight'], 3.14159265359) + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_set_weight_ipv4_new_format(self): + self.create_sample_ring() + # Test ipv4(new format) + argv = \ + ["", self.tmpfile, "set_weight", + "--id", "0", "--region", "0", "--zone", "0", + "--ip", "127.0.0.1", + "--port", "6000", + "--replication-ip", "127.0.0.1", + "--replication-port", "6000", + "--device", "sda1", "--meta", "some meta data", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + + # Check that weight was changed + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['weight'], 3.14159265359) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['weight'], 100) + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_set_weight_ipv6_new_format(self): + self.create_sample_ring() + # add IPV6 + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "[2001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[2::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "100"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test ipv6(new format) + argv = \ + ["", self.tmpfile, "set_weight", + "--id", "2", "--region", "2", "--zone", "3", + "--ip", "[2001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[2::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['weight'], 100) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['weight'], 100) + + # Check that weight was changed + dev = [d for d in ring.devs if d['id'] == 2][0] + self.assertEqual(dev['weight'], 3.14159265359) + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_set_weight_domain_new_format(self): + self.create_sample_ring() + # add domain name + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "100"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test domain name + argv = \ + ["", self.tmpfile, "set_weight", + "--id", "2", "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['weight'], 100) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['weight'], 100) + + # Check that weight was changed + dev = [d for d in ring.devs if d['id'] == 2][0] + self.assertEqual(dev['weight'], 3.14159265359) + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_set_weight_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["", self.tmpfile, "set_weight"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_set_weight_no_matching(self): + self.create_sample_ring() + # Test No matching devices + argv = ["", self.tmpfile, "set_weight", + "--ip", "unknown"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + def test_set_info(self): for search_value in self.search_values: self.create_sample_ring() argv = ["", self.tmpfile, "set_info", search_value, "127.0.1.1:8000/sda1_other meta data"] - self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) + self.assertRaises(SystemExit, ringbuilder.main, argv) # Check that device was created with given data ring = RingBuilder.load(self.tmpfile) @@ -218,38 +944,336 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin): ring.rebalance() self.assertTrue(ring.validate()) + def test_set_info_ipv4_old_format(self): + self.create_sample_ring() + # Test ipv4(old format) + argv = ["", self.tmpfile, "set_info", + "d0r0z0-127.0.0.1:6000R127.0.0.1:6000/sda1_some meta data", + "127.0.1.1:8000R127.0.1.1:8000/sda10_other meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Check that device was created with given data + ring = RingBuilder.load(self.tmpfile) + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['ip'], '127.0.1.1') + self.assertEqual(dev['port'], 8000) + self.assertEqual(dev['replication_ip'], '127.0.1.1') + self.assertEqual(dev['replication_port'], 8000) + self.assertEqual(dev['device'], 'sda10') + self.assertEqual(dev['meta'], 'other meta data') + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['ip'], '127.0.0.2') + self.assertEqual(dev['port'], 6001) + self.assertEqual(dev['device'], 'sda2') + self.assertEqual(dev['meta'], '') + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_set_info_ipv6_old_format(self): + self.create_sample_ring() + # add IPV6 + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "[2001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[2::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test ipv6(old format) + argv = ["", self.tmpfile, "set_info", + "d2r2z3-[2001:0000:1234:0000:0000:C1C0:ABCD:0876]:6000" + "R[2::10]:7000/sda3_some meta data", + "[3001:0000:1234:0000:0000:C1C0:ABCD:0876]:8000" + "R[3::10]:8000/sda30_other meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['ip'], '127.0.0.1') + self.assertEqual(dev['port'], 6000) + self.assertEqual(dev['replication_ip'], '127.0.0.1') + self.assertEqual(dev['replication_port'], 6000) + self.assertEqual(dev['device'], 'sda1') + self.assertEqual(dev['meta'], 'some meta data') + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['ip'], '127.0.0.2') + self.assertEqual(dev['port'], 6001) + self.assertEqual(dev['device'], 'sda2') + self.assertEqual(dev['meta'], '') + + # Check that device was created with given data + ring = RingBuilder.load(self.tmpfile) + dev = [d for d in ring.devs if d['id'] == 2][0] + self.assertEqual(dev['ip'], '3001:0:1234::c1c0:abcd:876') + self.assertEqual(dev['port'], 8000) + self.assertEqual(dev['replication_ip'], '3::10') + self.assertEqual(dev['replication_port'], 8000) + self.assertEqual(dev['device'], 'sda30') + self.assertEqual(dev['meta'], 'other meta data') + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_set_info_ipv4_new_format(self): + self.create_sample_ring() + # Test ipv4(new format) + argv = \ + ["", self.tmpfile, "set_info", + "--id", "0", "--region", "0", "--zone", "0", + "--ip", "127.0.0.1", + "--port", "6000", + "--replication-ip", "127.0.0.1", + "--replication-port", "6000", + "--device", "sda1", "--meta", "some meta data", + "--change-ip", "127.0.2.1", + "--change-port", "9000", + "--change-replication-ip", "127.0.2.1", + "--change-replication-port", "9000", + "--change-device", "sda100", "--change-meta", "other meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Check that device was created with given data + ring = RingBuilder.load(self.tmpfile) + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['ip'], '127.0.2.1') + self.assertEqual(dev['port'], 9000) + self.assertEqual(dev['replication_ip'], '127.0.2.1') + self.assertEqual(dev['replication_port'], 9000) + self.assertEqual(dev['device'], 'sda100') + self.assertEqual(dev['meta'], 'other meta data') + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['ip'], '127.0.0.2') + self.assertEqual(dev['port'], 6001) + self.assertEqual(dev['device'], 'sda2') + self.assertEqual(dev['meta'], '') + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_set_info_ipv6_new_format(self): + self.create_sample_ring() + # add IPV6 + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "[2001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[2::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test ipv6(new format) + argv = \ + ["", self.tmpfile, "set_info", + "--id", "2", "--region", "2", "--zone", "3", + "--ip", "[2001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[2::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--change-ip", "[4001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--change-port", "9000", + "--change-replication-ip", "[4::10]", + "--change-replication-port", "9000", + "--change-device", "sda300", "--change-meta", "other meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['ip'], '127.0.0.1') + self.assertEqual(dev['port'], 6000) + self.assertEqual(dev['replication_ip'], '127.0.0.1') + self.assertEqual(dev['replication_port'], 6000) + self.assertEqual(dev['device'], 'sda1') + self.assertEqual(dev['meta'], 'some meta data') + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['ip'], '127.0.0.2') + self.assertEqual(dev['port'], 6001) + self.assertEqual(dev['device'], 'sda2') + self.assertEqual(dev['meta'], '') + + # Check that device was created with given data + ring = RingBuilder.load(self.tmpfile) + dev = [d for d in ring.devs if d['id'] == 2][0] + self.assertEqual(dev['ip'], '4001:0:1234::c1c0:abcd:876') + self.assertEqual(dev['port'], 9000) + self.assertEqual(dev['replication_ip'], '4::10') + self.assertEqual(dev['replication_port'], 9000) + self.assertEqual(dev['device'], 'sda300') + self.assertEqual(dev['meta'], 'other meta data') + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_set_info_domain_new_format(self): + self.create_sample_ring() + # add domain name + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test domain name + argv = \ + ["", self.tmpfile, "set_info", + "--id", "2", "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--change-ip", "test.test2.com", + "--change-port", "9000", + "--change-replication-ip", "r.test2.com", + "--change-replication-port", "9000", + "--change-device", "sda300", "--change-meta", "other meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 0][0] + self.assertEqual(dev['ip'], '127.0.0.1') + self.assertEqual(dev['port'], 6000) + self.assertEqual(dev['replication_ip'], '127.0.0.1') + self.assertEqual(dev['replication_port'], 6000) + self.assertEqual(dev['device'], 'sda1') + self.assertEqual(dev['meta'], 'some meta data') + + # Check that second device in ring is not affected + dev = [d for d in ring.devs if d['id'] == 1][0] + self.assertEqual(dev['ip'], '127.0.0.2') + self.assertEqual(dev['port'], 6001) + self.assertEqual(dev['device'], 'sda2') + self.assertEqual(dev['meta'], '') + + # Check that device was created with given data + ring = RingBuilder.load(self.tmpfile) + dev = [d for d in ring.devs if d['id'] == 2][0] + self.assertEqual(dev['ip'], 'test.test2.com') + self.assertEqual(dev['port'], 9000) + self.assertEqual(dev['replication_ip'], 'r.test2.com') + self.assertEqual(dev['replication_port'], 9000) + self.assertEqual(dev['device'], 'sda300') + self.assertEqual(dev['meta'], 'other meta data') + + # Final check, rebalance and check ring is ok + ring.rebalance() + self.assertTrue(ring.validate()) + + def test_set_info_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["", self.tmpfile, "set_info"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_set_info_no_matching(self): + self.create_sample_ring() + # Test No matching devices + argv = ["", self.tmpfile, "set_info", + "--ip", "unknown"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_set_info_already_exists(self): + self.create_sample_ring() + # Test Set a device that already exists + argv = \ + ["", self.tmpfile, "set_info", + "--id", "0", "--region", "0", "--zone", "0", + "--ip", "127.0.0.1", + "--port", "6000", + "--replication-ip", "127.0.0.1", + "--replication-port", "6000", + "--device", "sda1", "--meta", "some meta data", + "--change-ip", "127.0.0.2", + "--change-port", "6001", + "--change-replication-ip", "127.0.0.2", + "--change-replication-port", "6001", + "--change-device", "sda2", "--change-meta", ""] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + def test_set_min_part_hours(self): self.create_sample_ring() argv = ["", self.tmpfile, "set_min_part_hours", "24"] - self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) + self.assertRaises(SystemExit, ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.min_part_hours, 24) + def test_set_min_part_hours_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["", self.tmpfile, "set_min_part_hours"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + def test_set_replicas(self): self.create_sample_ring() argv = ["", self.tmpfile, "set_replicas", "3.14159265359"] - self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) + self.assertRaises(SystemExit, ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.replicas, 3.14159265359) def test_set_overload(self): self.create_sample_ring() argv = ["", self.tmpfile, "set_overload", "0.19878"] - self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) + self.assertRaises(SystemExit, ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.overload, 0.19878) def test_set_overload_negative(self): self.create_sample_ring() argv = ["", self.tmpfile, "set_overload", "-0.19878"] - self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) + self.assertRaises(SystemExit, ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.overload, 0.0) def test_set_overload_non_numeric(self): self.create_sample_ring() argv = ["", self.tmpfile, "set_overload", "swedish fish"] - self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) + self.assertRaises(SystemExit, ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.overload, 0.0) @@ -289,21 +1313,52 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin): self.assertTrue('1000.00%' in out) self.assertTrue('10.000000' in out) + def test_set_replicas_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["", self.tmpfile, "set_replicas"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_set_replicas_invalid_value(self): + # Test not a valid number + argv = ["", self.tmpfile, "set_replicas", "test"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + # Test new replicas is 0 + argv = ["", self.tmpfile, "set_replicas", "0"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + def test_validate(self): self.create_sample_ring() ring = RingBuilder.load(self.tmpfile) ring.rebalance() ring.save(self.tmpfile) argv = ["", self.tmpfile, "validate"] - self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) + self.assertRaises(SystemExit, ringbuilder.main, argv) def test_validate_empty_file(self): open(self.tmpfile, 'a').close argv = ["", self.tmpfile, "validate"] + err = None try: - swift.cli.ringbuilder.main(argv) + ringbuilder.main(argv) except SystemExit as e: - self.assertEquals(e.code, 2) + err = e + self.assertEquals(err.code, 2) def test_validate_corrupted_file(self): self.create_sample_ring() @@ -316,38 +1371,338 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin): # corrupt the file with open(self.tmpfile, 'wb') as f: f.write(os.urandom(1024)) + err = None try: - swift.cli.ringbuilder.main(argv) + ringbuilder.main(argv) except SystemExit as e: - self.assertEquals(e.code, 2) + err = e + self.assertEquals(err.code, 2) def test_validate_non_existent_file(self): rand_file = '%s/%s' % ('/tmp', str(uuid.uuid4())) argv = ["", rand_file, "validate"] + err = None try: - swift.cli.ringbuilder.main(argv) + ringbuilder.main(argv) except SystemExit as e: - self.assertEquals(e.code, 2) + err = e + self.assertEquals(err.code, 2) def test_validate_non_accessible_file(self): with mock.patch.object( RingBuilder, 'load', mock.Mock(side_effect=exceptions.PermissionError)): argv = ["", self.tmpfile, "validate"] + err = None try: - swift.cli.ringbuilder.main(argv) + ringbuilder.main(argv) except SystemExit as e: - self.assertEquals(e.code, 2) + err = e + self.assertEquals(err.code, 2) def test_validate_generic_error(self): with mock.patch.object( RingBuilder, 'load', mock.Mock( side_effect=IOError('Generic error occurred'))): argv = ["", self.tmpfile, "validate"] + err = None try: - swift.cli.ringbuilder.main(argv) + ringbuilder.main(argv) except SystemExit as e: - self.assertEquals(e.code, 2) + err = e + self.assertEquals(err.code, 2) + + def test_search_device_ipv4_old_format(self): + self.create_sample_ring() + # Test ipv4(old format) + argv = ["", self.tmpfile, "search", + "d0r0z0-127.0.0.1:6000R127.0.0.1:6000/sda1_some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + def test_search_device_ipv6_old_format(self): + self.create_sample_ring() + # add IPV6 + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "[2001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[2::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test ipv6(old format) + argv = ["", self.tmpfile, "search", + "d2r2z3-[2001:0000:1234:0000:0000:C1C0:ABCD:0876]:6000" + "R[2::10]:7000/sda3_some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + def test_search_device_ipv4_new_format(self): + self.create_sample_ring() + # Test ipv4(new format) + argv = \ + ["", self.tmpfile, "search", + "--id", "0", "--region", "0", "--zone", "0", + "--ip", "127.0.0.1", + "--port", "6000", + "--replication-ip", "127.0.0.1", + "--replication-port", "6000", + "--device", "sda1", "--meta", "some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + def test_search_device_ipv6_new_format(self): + self.create_sample_ring() + # add IPV6 + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "[2001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[2::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test ipv6(new format) + argv = \ + ["", self.tmpfile, "search", + "--id", "2", "--region", "2", "--zone", "3", + "--ip", "[2001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[2::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + def test_search_device_domain_new_format(self): + self.create_sample_ring() + # add domain name + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test domain name + argv = \ + ["", self.tmpfile, "search", + "--id", "2", "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + def test_search_device_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["", self.tmpfile, "search"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_search_device_no_matching(self): + self.create_sample_ring() + # Test No matching devices + argv = ["", self.tmpfile, "search", + "--ip", "unknown"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_list_parts_ipv4_old_format(self): + self.create_sample_ring() + # Test ipv4(old format) + argv = ["", self.tmpfile, "list_parts", + "d0r0z0-127.0.0.1:6000R127.0.0.1:6000/sda1_some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + def test_list_parts_ipv6_old_format(self): + self.create_sample_ring() + # add IPV6 + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "[2001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[2::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test ipv6(old format) + argv = ["", self.tmpfile, "list_parts", + "d2r2z3-[2001:0000:1234:0000:0000:C1C0:ABCD:0876]:6000" + "R[2::10]:7000/sda3_some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + def test_list_parts_ipv4_new_format(self): + self.create_sample_ring() + # Test ipv4(new format) + argv = \ + ["", self.tmpfile, "list_parts", + "--id", "0", "--region", "0", "--zone", "0", + "--ip", "127.0.0.1", + "--port", "6000", + "--replication-ip", "127.0.0.1", + "--replication-port", "6000", + "--device", "sda1", "--meta", "some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + def test_list_parts_ipv6_new_format(self): + self.create_sample_ring() + # add IPV6 + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "[2001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[2::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test ipv6(new format) + argv = \ + ["", self.tmpfile, "list_parts", + "--id", "2", "--region", "2", "--zone", "3", + "--ip", "[2001:0000:1234:0000:0000:C1C0:ABCD:0876]", + "--port", "6000", + "--replication-ip", "[2::10]", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + def test_list_parts_domain_new_format(self): + self.create_sample_ring() + # add domain name + argv = \ + ["", self.tmpfile, "add", + "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data", + "--weight", "3.14159265359"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + # Test domain name + argv = \ + ["", self.tmpfile, "list_parts", + "--id", "2", "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", "--meta", "some meta data"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + def test_list_parts_number_of_arguments(self): + # Test Number of arguments abnormal + argv = ["", self.tmpfile, "list_parts"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_list_parts_no_matching(self): + self.create_sample_ring() + # Test No matching devices + argv = ["", self.tmpfile, "list_parts", + "--ip", "unknown"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_unknown(self): + argv = ["", self.tmpfile, "unknown"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_default(self): + self.create_sample_ring() + argv = ["", self.tmpfile] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + def test_rebalance(self): + self.create_sample_ring() + argv = ["", self.tmpfile, "rebalance", "3"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + ring = RingBuilder.load(self.tmpfile) + self.assertTrue(ring.validate()) + + def test_rebalance_no_device_change(self): + self.create_sample_ring() + ring = RingBuilder.load(self.tmpfile) + ring.rebalance() + ring.save(self.tmpfile) + # Test No change to the device + argv = ["", self.tmpfile, "rebalance", "3"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 1) + + def test_rebalance_no_devices(self): + # Test no devices + argv = ["", self.tmpfile, "create", "6", "3.14159265359", "1"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + argv = ["", self.tmpfile, "rebalance"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) + + def test_write_ring(self): + self.create_sample_ring() + argv = ["", self.tmpfile, "rebalance"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + argv = ["", self.tmpfile, "write_ring"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + + def test_write_builder(self): + # Test builder file already exists + self.create_sample_ring() + argv = ["", self.tmpfile, "rebalance"] + self.assertRaises(SystemExit, ringbuilder.main, argv) + argv = ["", self.tmpfile, "write_builder"] + err = None + try: + ringbuilder.main(argv) + except SystemExit as e: + err = e + self.assertEquals(err.code, 2) def test_warn_at_risk(self): self.create_sample_ring() @@ -356,7 +1711,7 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin): ring.save(self.tmpfile) argv = ["", self.tmpfile, "rebalance"] try: - swift.cli.ringbuilder.main(argv) + ringbuilder.main(argv) except SystemExit as e: self.assertEquals(e.code, 1) @@ -374,6 +1729,21 @@ class TestRebalanceCommand(unittest.TestCase, RunSwiftRingBuilderMixin): except OSError: pass + def run_srb(self, *argv): + mock_stdout = StringIO.StringIO() + mock_stderr = StringIO.StringIO() + + srb_args = ["", self.tempfile] + [str(s) for s in argv] + + try: + with mock.patch("sys.stdout", mock_stdout): + with mock.patch("sys.stderr", mock_stderr): + ringbuilder.main(srb_args) + except SystemExit as err: + if err.code not in (0, 1): # (success, warning) + raise + return (mock_stdout.getvalue(), mock_stderr.getvalue()) + def test_rebalance_warning_appears(self): self.run_srb("create", 8, 3, 24) # all in one machine: totally balanceable diff --git a/test/unit/common/ring/test_utils.py b/test/unit/common/ring/test_utils.py index 0a7cae2828..a112547310 100644 --- a/test/unit/common/ring/test_utils.py +++ b/test/unit/common/ring/test_utils.py @@ -16,11 +16,17 @@ import unittest from swift.common import ring -from swift.common.ring.utils import (build_tier_tree, tiers_for_dev, - parse_search_value, parse_args, - build_dev_from_opts, find_parts, +from swift.common.ring.utils import (tiers_for_dev, build_tier_tree, + validate_and_normalize_ip, + validate_and_normalize_address, + is_valid_ip, is_valid_ipv4, + is_valid_ipv6, is_valid_hostname, + is_local_device, parse_search_value, + parse_search_values_from_opts, + parse_change_values_from_opts, + validate_args, parse_args, parse_builder_ring_filename_args, - dispersion_report) + build_dev_from_opts, dispersion_report) class TestUtils(unittest.TestCase): @@ -95,6 +101,121 @@ class TestUtils(unittest.TestCase): (1, 2, '192.168.2.2:6000', 10), (1, 2, '192.168.2.2:6000', 11)])) + def test_is_valid_ip(self): + self.assertTrue(is_valid_ip("127.0.0.1")) + self.assertTrue(is_valid_ip("10.0.0.1")) + ipv6 = "fe80:0000:0000:0000:0204:61ff:fe9d:f156" + self.assertTrue(is_valid_ip(ipv6)) + ipv6 = "fe80:0:0:0:204:61ff:fe9d:f156" + self.assertTrue(is_valid_ip(ipv6)) + ipv6 = "fe80::204:61ff:fe9d:f156" + self.assertTrue(is_valid_ip(ipv6)) + ipv6 = "fe80:0000:0000:0000:0204:61ff:254.157.241.86" + self.assertTrue(is_valid_ip(ipv6)) + ipv6 = "fe80:0:0:0:0204:61ff:254.157.241.86" + self.assertTrue(is_valid_ip(ipv6)) + ipv6 = "fe80::204:61ff:254.157.241.86" + self.assertTrue(is_valid_ip(ipv6)) + ipv6 = "fe80::" + self.assertTrue(is_valid_ip(ipv6)) + ipv6 = "::1" + self.assertTrue(is_valid_ip(ipv6)) + not_ipv6 = "3ffe:0b00:0000:0001:0000:0000:000a" + self.assertFalse(is_valid_ip(not_ipv6)) + not_ipv6 = "1:2:3:4:5:6::7:8" + self.assertFalse(is_valid_ip(not_ipv6)) + + def test_is_valid_ipv4(self): + self.assertTrue(is_valid_ipv4("127.0.0.1")) + self.assertTrue(is_valid_ipv4("10.0.0.1")) + ipv6 = "fe80:0000:0000:0000:0204:61ff:fe9d:f156" + self.assertFalse(is_valid_ipv4(ipv6)) + ipv6 = "fe80:0:0:0:204:61ff:fe9d:f156" + self.assertFalse(is_valid_ipv4(ipv6)) + ipv6 = "fe80::204:61ff:fe9d:f156" + self.assertFalse(is_valid_ipv4(ipv6)) + ipv6 = "fe80:0000:0000:0000:0204:61ff:254.157.241.86" + self.assertFalse(is_valid_ipv4(ipv6)) + ipv6 = "fe80:0:0:0:0204:61ff:254.157.241.86" + self.assertFalse(is_valid_ipv4(ipv6)) + ipv6 = "fe80::204:61ff:254.157.241.86" + self.assertFalse(is_valid_ipv4(ipv6)) + ipv6 = "fe80::" + self.assertFalse(is_valid_ipv4(ipv6)) + ipv6 = "::1" + self.assertFalse(is_valid_ipv4(ipv6)) + not_ipv6 = "3ffe:0b00:0000:0001:0000:0000:000a" + self.assertFalse(is_valid_ipv4(not_ipv6)) + not_ipv6 = "1:2:3:4:5:6::7:8" + self.assertFalse(is_valid_ipv4(not_ipv6)) + + def test_is_valid_ipv6(self): + self.assertFalse(is_valid_ipv6("127.0.0.1")) + self.assertFalse(is_valid_ipv6("10.0.0.1")) + ipv6 = "fe80:0000:0000:0000:0204:61ff:fe9d:f156" + self.assertTrue(is_valid_ipv6(ipv6)) + ipv6 = "fe80:0:0:0:204:61ff:fe9d:f156" + self.assertTrue(is_valid_ipv6(ipv6)) + ipv6 = "fe80::204:61ff:fe9d:f156" + self.assertTrue(is_valid_ipv6(ipv6)) + ipv6 = "fe80:0000:0000:0000:0204:61ff:254.157.241.86" + self.assertTrue(is_valid_ipv6(ipv6)) + ipv6 = "fe80:0:0:0:0204:61ff:254.157.241.86" + self.assertTrue(is_valid_ipv6(ipv6)) + ipv6 = "fe80::204:61ff:254.157.241.86" + self.assertTrue(is_valid_ipv6(ipv6)) + ipv6 = "fe80::" + self.assertTrue(is_valid_ipv6(ipv6)) + ipv6 = "::1" + self.assertTrue(is_valid_ipv6(ipv6)) + not_ipv6 = "3ffe:0b00:0000:0001:0000:0000:000a" + self.assertFalse(is_valid_ipv6(not_ipv6)) + not_ipv6 = "1:2:3:4:5:6::7:8" + self.assertFalse(is_valid_ipv6(not_ipv6)) + + def test_is_valid_hostname(self): + self.assertTrue(is_valid_hostname("local")) + self.assertTrue(is_valid_hostname("test.test.com")) + hostname = "test." * 51 + self.assertTrue(is_valid_hostname(hostname)) + hostname = hostname.rstrip('.') + self.assertTrue(is_valid_hostname(hostname)) + hostname = hostname + "00" + self.assertFalse(is_valid_hostname(hostname)) + self.assertFalse(is_valid_hostname("$blah#")) + + def test_is_local_device(self): + my_ips = ["127.0.0.1", + "0000:0000:0000:0000:0000:0000:0000:0001"] + my_port = 6000 + self.assertTrue(is_local_device(my_ips, my_port, + "localhost", + my_port)) + + def test_validate_and_normalize_ip(self): + ipv4 = "10.0.0.1" + self.assertEqual(ipv4, validate_and_normalize_ip(ipv4)) + ipv6 = "fe80::204:61ff:fe9d:f156" + self.assertEqual(ipv6, validate_and_normalize_ip(ipv6.upper())) + hostname = "test.test.com" + self.assertRaises(ValueError, + validate_and_normalize_ip, hostname) + hostname = "$blah#" + self.assertRaises(ValueError, + validate_and_normalize_ip, hostname) + + def test_validate_and_normalize_address(self): + ipv4 = "10.0.0.1" + self.assertEqual(ipv4, validate_and_normalize_address(ipv4)) + ipv6 = "fe80::204:61ff:fe9d:f156" + self.assertEqual(ipv6, validate_and_normalize_address(ipv6.upper())) + hostname = "test.test.com" + self.assertEqual(hostname, + validate_and_normalize_address(hostname.upper())) + hostname = "$blah#" + self.assertRaises(ValueError, + validate_and_normalize_address, hostname) + def test_parse_search_value(self): res = parse_search_value('r0') self.assertEqual(res, {'region': 0}) @@ -108,6 +229,8 @@ class TestUtils(unittest.TestCase): self.assertEqual(res, {'zone': 1}) res = parse_search_value('-127.0.0.1') self.assertEqual(res, {'ip': '127.0.0.1'}) + res = parse_search_value('127.0.0.1') + self.assertEqual(res, {'ip': '127.0.0.1'}) res = parse_search_value('-[127.0.0.1]:10001') self.assertEqual(res, {'ip': '127.0.0.1', 'port': 10001}) res = parse_search_value(':10001') @@ -125,22 +248,268 @@ class TestUtils(unittest.TestCase): self.assertEqual(res, {'meta': 'meta1'}) self.assertRaises(ValueError, parse_search_value, 'OMGPONIES') - def test_replication_defaults(self): - args = '-r 1 -z 1 -i 127.0.0.1 -p 6010 -d d1 -w 100'.split() - opts, _ = parse_args(args) - device = build_dev_from_opts(opts) + def test_parse_search_values_from_opts(self): + argv = \ + ["--id", "1", "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", + "--meta", "some meta data", + "--weight", "3.14159265359", + "--change-ip", "change.test.test.com", + "--change-port", "6001", + "--change-replication-ip", "change.r.test.com", + "--change-replication-port", "7001", + "--change-device", "sdb3", + "--change-meta", "some meta data for change"] expected = { - 'device': 'd1', - 'ip': '127.0.0.1', - 'meta': '', - 'port': 6010, - 'region': 1, - 'replication_ip': '127.0.0.1', - 'replication_port': 6010, - 'weight': 100.0, - 'zone': 1, + 'id': 1, + 'region': 2, + 'zone': 3, + 'ip': "test.test.com", + 'port': 6000, + 'replication_ip': "r.test.com", + 'replication_port': 7000, + 'device': "sda3", + 'meta': "some meta data", + 'weight': 3.14159265359, } - self.assertEquals(device, expected) + new_cmd_format, opts, args = validate_args(argv) + search_values = parse_search_values_from_opts(opts) + self.assertEquals(search_values, expected) + + argv = \ + ["--id", "1", "--region", "2", "--zone", "3", + "--ip", "127.0.0.1", + "--port", "6000", + "--replication-ip", "127.0.0.10", + "--replication-port", "7000", + "--device", "sda3", + "--meta", "some meta data", + "--weight", "3.14159265359", + "--change-ip", "127.0.0.2", + "--change-port", "6001", + "--change-replication-ip", "127.0.0.20", + "--change-replication-port", "7001", + "--change-device", "sdb3", + "--change-meta", "some meta data for change"] + expected = { + 'id': 1, + 'region': 2, + 'zone': 3, + 'ip': "127.0.0.1", + 'port': 6000, + 'replication_ip': "127.0.0.10", + 'replication_port': 7000, + 'device': "sda3", + 'meta': "some meta data", + 'weight': 3.14159265359, + } + new_cmd_format, opts, args = validate_args(argv) + search_values = parse_search_values_from_opts(opts) + self.assertEquals(search_values, expected) + + argv = \ + ["--id", "1", "--region", "2", "--zone", "3", + "--ip", "[127.0.0.1]", + "--port", "6000", + "--replication-ip", "[127.0.0.10]", + "--replication-port", "7000", + "--device", "sda3", + "--meta", "some meta data", + "--weight", "3.14159265359", + "--change-ip", "[127.0.0.2]", + "--change-port", "6001", + "--change-replication-ip", "[127.0.0.20]", + "--change-replication-port", "7001", + "--change-device", "sdb3", + "--change-meta", "some meta data for change"] + new_cmd_format, opts, args = validate_args(argv) + search_values = parse_search_values_from_opts(opts) + self.assertEquals(search_values, expected) + + def test_parse_change_values_from_opts(self): + argv = \ + ["--id", "1", "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", + "--meta", "some meta data", + "--weight", "3.14159265359", + "--change-ip", "change.test.test.com", + "--change-port", "6001", + "--change-replication-ip", "change.r.test.com", + "--change-replication-port", "7001", + "--change-device", "sdb3", + "--change-meta", "some meta data for change"] + expected = { + 'ip': "change.test.test.com", + 'port': 6001, + 'replication_ip': "change.r.test.com", + 'replication_port': 7001, + 'device': "sdb3", + 'meta': "some meta data for change", + } + new_cmd_format, opts, args = validate_args(argv) + search_values = parse_change_values_from_opts(opts) + self.assertEquals(search_values, expected) + + argv = \ + ["--id", "1", "--region", "2", "--zone", "3", + "--ip", "127.0.0.1", + "--port", "6000", + "--replication-ip", "127.0.0.10", + "--replication-port", "7000", + "--device", "sda3", + "--meta", "some meta data", + "--weight", "3.14159265359", + "--change-ip", "127.0.0.2", + "--change-port", "6001", + "--change-replication-ip", "127.0.0.20", + "--change-replication-port", "7001", + "--change-device", "sdb3", + "--change-meta", "some meta data for change"] + expected = { + 'ip': "127.0.0.2", + 'port': 6001, + 'replication_ip': "127.0.0.20", + 'replication_port': 7001, + 'device': "sdb3", + 'meta': "some meta data for change", + } + new_cmd_format, opts, args = validate_args(argv) + search_values = parse_change_values_from_opts(opts) + self.assertEquals(search_values, expected) + + argv = \ + ["--id", "1", "--region", "2", "--zone", "3", + "--ip", "[127.0.0.1]", + "--port", "6000", + "--replication-ip", "[127.0.0.10]", + "--replication-port", "7000", + "--device", "sda3", + "--meta", "some meta data", + "--weight", "3.14159265359", + "--change-ip", "[127.0.0.2]", + "--change-port", "6001", + "--change-replication-ip", "[127.0.0.20]", + "--change-replication-port", "7001", + "--change-device", "sdb3", + "--change-meta", "some meta data for change"] + new_cmd_format, opts, args = validate_args(argv) + search_values = parse_change_values_from_opts(opts) + self.assertEquals(search_values, expected) + + def test_validate_args(self): + argv = \ + ["--id", "1", "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", + "--meta", "some meta data", + "--weight", "3.14159265359", + "--change-ip", "change.test.test.com", + "--change-port", "6001", + "--change-replication-ip", "change.r.test.com", + "--change-replication-port", "7001", + "--change-device", "sdb3", + "--change-meta", "some meta data for change"] + new_cmd_format, opts, args = validate_args(argv) + self.assertTrue(new_cmd_format) + self.assertEqual(opts.id, 1) + self.assertEqual(opts.region, 2) + self.assertEqual(opts.zone, 3) + self.assertEqual(opts.ip, "test.test.com") + self.assertEqual(opts.port, 6000) + self.assertEqual(opts.replication_ip, "r.test.com") + self.assertEqual(opts.replication_port, 7000) + self.assertEqual(opts.device, "sda3") + self.assertEqual(opts.meta, "some meta data") + self.assertEqual(opts.weight, 3.14159265359) + self.assertEqual(opts.change_ip, "change.test.test.com") + self.assertEqual(opts.change_port, 6001) + self.assertEqual(opts.change_replication_ip, "change.r.test.com") + self.assertEqual(opts.change_replication_port, 7001) + self.assertEqual(opts.change_device, "sdb3") + self.assertEqual(opts.change_meta, "some meta data for change") + + argv = \ + ["--id", "0", "--region", "0", "--zone", "0", + "--ip", "", + "--port", "0", + "--replication-ip", "", + "--replication-port", "0", + "--device", "", + "--meta", "", + "--weight", "0", + "--change-ip", "", + "--change-port", "0", + "--change-replication-ip", "", + "--change-replication-port", "0", + "--change-device", "", + "--change-meta", ""] + new_cmd_format, opts, args = validate_args(argv) + self.assertFalse(new_cmd_format) + + argv = \ + ["--id", "0", "--region", "0", "--zone", "0", + "--ip", "", + "--port", "0", + "--replication-ip", "", + "--replication-port", "0", + "--device", "", + "--meta", "", + "--weight", "0", + "--change-ip", "change.test.test.com", + "--change-port", "6001", + "--change-replication-ip", "change.r.test.com", + "--change-replication-port", "7001", + "--change-device", "sdb3", + "--change-meta", "some meta data for change"] + new_cmd_format, opts, args = validate_args(argv) + self.assertFalse(new_cmd_format) + + def test_parse_args(self): + argv = \ + ["--id", "1", "--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", + "--meta", "some meta data", + "--weight", "3.14159265359", + "--change-ip", "change.test.test.com", + "--change-port", "6001", + "--change-replication-ip", "change.r.test.com", + "--change-replication-port", "7001", + "--change-device", "sdb3", + "--change-meta", "some meta data for change"] + + opts, args = parse_args(argv) + self.assertEqual(opts.id, 1) + self.assertEqual(opts.region, 2) + self.assertEqual(opts.zone, 3) + self.assertEqual(opts.ip, "test.test.com") + self.assertEqual(opts.port, 6000) + self.assertEqual(opts.replication_ip, "r.test.com") + self.assertEqual(opts.replication_port, 7000) + self.assertEqual(opts.device, "sda3") + self.assertEqual(opts.meta, "some meta data") + self.assertEqual(opts.weight, 3.14159265359) + self.assertEqual(opts.change_ip, "change.test.test.com") + self.assertEqual(opts.change_port, 6001) + self.assertEqual(opts.change_replication_ip, "change.r.test.com") + self.assertEqual(opts.change_replication_port, 7001) + self.assertEqual(opts.change_device, "sdb3") + self.assertEqual(opts.change_meta, "some meta data for change") + self.assertEqual(len(args), 0) def test_parse_builder_ring_filename_args(self): args = 'swift-ring-builder object.builder write_ring' @@ -161,33 +530,86 @@ class TestUtils(unittest.TestCase): 'my.file.name', 'my.file.name.ring.gz' ), parse_builder_ring_filename_args(args.split())) - def test_find_parts(self): - rb = ring.RingBuilder(8, 3, 0) - rb.add_dev({'id': 0, 'region': 1, 'zone': 0, 'weight': 100, - 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) - rb.add_dev({'id': 1, 'region': 1, 'zone': 1, 'weight': 100, - 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) - rb.add_dev({'id': 2, 'region': 1, 'zone': 2, 'weight': 100, - 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) - rb.rebalance() + def test_build_dev_from_opts(self): + argv = \ + ["--region", "2", "--zone", "3", + "--ip", "test.test.com", + "--port", "6000", + "--replication-ip", "r.test.com", + "--replication-port", "7000", + "--device", "sda3", + "--meta", "some meta data", + "--weight", "3.14159265359"] + expected = { + 'region': 2, + 'zone': 3, + 'ip': "test.test.com", + 'port': 6000, + 'replication_ip': "r.test.com", + 'replication_port': 7000, + 'device': "sda3", + 'meta': "some meta data", + 'weight': 3.14159265359, + } + opts, args = parse_args(argv) + device = build_dev_from_opts(opts) + self.assertEquals(device, expected) - rb.add_dev({'id': 3, 'region': 2, 'zone': 1, 'weight': 10, - 'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'}) - rb.pretend_min_part_hours_passed() - rb.rebalance() + argv = \ + ["--region", "2", "--zone", "3", + "--ip", "[test.test.com]", + "--port", "6000", + "--replication-ip", "[r.test.com]", + "--replication-port", "7000", + "--device", "sda3", + "--meta", "some meta data", + "--weight", "3.14159265359"] + opts, args = parse_args(argv) + self.assertRaises(ValueError, build_dev_from_opts, opts) - argv = ['swift-ring-builder', 'object.builder', - 'list_parts', '127.0.0.1'] - sorted_partition_count = find_parts(rb, argv) + argv = \ + ["--region", "2", "--zone", "3", + "--ip", "[test.test.com]", + "--port", "6000", + "--replication-ip", "[r.test.com]", + "--replication-port", "7000", + "--meta", "some meta data", + "--weight", "3.14159265359"] + opts, args = parse_args(argv) + self.assertRaises(ValueError, build_dev_from_opts, opts) - # Expect 256 partitions in the output - self.assertEqual(256, len(sorted_partition_count)) + def test_replication_defaults(self): + args = '-r 1 -z 1 -i 127.0.0.1 -p 6010 -d d1 -w 100'.split() + opts, _ = parse_args(args) + device = build_dev_from_opts(opts) + expected = { + 'device': 'd1', + 'ip': '127.0.0.1', + 'meta': '', + 'port': 6010, + 'region': 1, + 'replication_ip': '127.0.0.1', + 'replication_port': 6010, + 'weight': 100.0, + 'zone': 1, + } + self.assertEquals(device, expected) - # Each partitions should have 3 replicas - for partition, count in sorted_partition_count: - self.assertEqual( - 3, count, "Partition %d has only %d replicas" % - (partition, count)) + args = '-r 1 -z 1 -i test.com -p 6010 -d d1 -w 100'.split() + opts, _ = parse_args(args) + device = build_dev_from_opts(opts) + expected = { + 'device': 'd1', + 'ip': 'test.com', + 'meta': '', + 'port': 6010, + 'region': 1, + 'replication_ip': 'test.com', + 'replication_port': 6010, + 'weight': 100.0, + 'zone': 1, + } + self.assertEquals(device, expected) def test_dispersion_report(self): rb = ring.RingBuilder(8, 3, 0) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index dce0d76aba..2b42fb52f3 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -1455,6 +1455,15 @@ class TestUtils(unittest.TestCase): self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'), 'objects/1/DEF/ABCDEF') + def test_expand_ipv6(self): + expanded_ipv6 = "fe80::204:61ff:fe9d:f156" + upper_ipv6 = "fe80:0000:0000:0000:0204:61ff:fe9d:f156" + self.assertEqual(expanded_ipv6, utils.expand_ipv6(upper_ipv6)) + omit_ipv6 = "fe80:0000:0000::0204:61ff:fe9d:f156" + self.assertEqual(expanded_ipv6, utils.expand_ipv6(omit_ipv6)) + less_num_ipv6 = "fe80:0:00:000:0204:61ff:fe9d:f156" + self.assertEqual(expanded_ipv6, utils.expand_ipv6(less_num_ipv6)) + def test_whataremyips(self): myips = utils.whataremyips() self.assert_(len(myips) > 1) @@ -2935,9 +2944,9 @@ class TestSwiftInfo(unittest.TestCase): utils._swift_info = {'swift': {'foo': 'bar'}, 'cap1': cap1} # expect no exceptions - info = utils.get_swift_info(disallowed_sections= - ['cap2.cap1_foo', 'cap1.no_match', - 'cap1.cap1_foo.no_match.no_match']) + info = utils.get_swift_info( + disallowed_sections=['cap2.cap1_foo', 'cap1.no_match', + 'cap1.cap1_foo.no_match.no_match']) self.assertEquals(info['cap1'], cap1)