791 lines
33 KiB
Python
791 lines
33 KiB
Python
# Copyright (c) 2010-2012 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import sys
|
|
import unittest
|
|
from collections import defaultdict
|
|
|
|
from swift.common import exceptions
|
|
from swift.common import ring
|
|
from swift.common.ring.utils import (tiers_for_dev, build_tier_tree,
|
|
validate_and_normalize_ip,
|
|
validate_and_normalize_address,
|
|
is_valid_hostname,
|
|
is_local_device, parse_search_value,
|
|
parse_search_values_from_opts,
|
|
parse_change_values_from_opts,
|
|
validate_args, parse_args,
|
|
parse_builder_ring_filename_args,
|
|
build_dev_from_opts, dispersion_report,
|
|
parse_address, get_tier_name, pretty_dev,
|
|
validate_replicas_by_tier)
|
|
|
|
|
|
class TestUtils(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.test_dev = {'region': 1, 'zone': 1, 'ip': '192.168.1.1',
|
|
'port': '6200', 'id': 0}
|
|
|
|
def get_test_devs():
|
|
dev0 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1',
|
|
'port': '6200', 'id': 0}
|
|
dev1 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1',
|
|
'port': '6200', 'id': 1}
|
|
dev2 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1',
|
|
'port': '6200', 'id': 2}
|
|
dev3 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2',
|
|
'port': '6200', 'id': 3}
|
|
dev4 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2',
|
|
'port': '6200', 'id': 4}
|
|
dev5 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2',
|
|
'port': '6200', 'id': 5}
|
|
dev6 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1',
|
|
'port': '6200', 'id': 6}
|
|
dev7 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1',
|
|
'port': '6200', 'id': 7}
|
|
dev8 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1',
|
|
'port': '6200', 'id': 8}
|
|
dev9 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2',
|
|
'port': '6200', 'id': 9}
|
|
dev10 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2',
|
|
'port': '6200', 'id': 10}
|
|
dev11 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2',
|
|
'port': '6200', 'id': 11}
|
|
return [dev0, dev1, dev2, dev3, dev4, dev5,
|
|
dev6, dev7, dev8, dev9, dev10, dev11]
|
|
|
|
self.test_devs = get_test_devs()
|
|
|
|
def test_tiers_for_dev(self):
|
|
self.assertEqual(
|
|
tiers_for_dev(self.test_dev),
|
|
((1,),
|
|
(1, 1),
|
|
(1, 1, '192.168.1.1'),
|
|
(1, 1, '192.168.1.1', 0)))
|
|
|
|
def test_build_tier_tree(self):
|
|
ret = build_tier_tree(self.test_devs)
|
|
self.assertEqual(len(ret), 8)
|
|
self.assertEqual(ret[()], set([(1,)]))
|
|
self.assertEqual(ret[(1,)], set([(1, 1), (1, 2)]))
|
|
self.assertEqual(ret[(1, 1)],
|
|
set([(1, 1, '192.168.1.2'),
|
|
(1, 1, '192.168.1.1')]))
|
|
self.assertEqual(ret[(1, 2)],
|
|
set([(1, 2, '192.168.2.2'),
|
|
(1, 2, '192.168.2.1')]))
|
|
self.assertEqual(ret[(1, 1, '192.168.1.1')],
|
|
set([(1, 1, '192.168.1.1', 0),
|
|
(1, 1, '192.168.1.1', 1),
|
|
(1, 1, '192.168.1.1', 2)]))
|
|
self.assertEqual(ret[(1, 1, '192.168.1.2')],
|
|
set([(1, 1, '192.168.1.2', 3),
|
|
(1, 1, '192.168.1.2', 4),
|
|
(1, 1, '192.168.1.2', 5)]))
|
|
self.assertEqual(ret[(1, 2, '192.168.2.1')],
|
|
set([(1, 2, '192.168.2.1', 6),
|
|
(1, 2, '192.168.2.1', 7),
|
|
(1, 2, '192.168.2.1', 8)]))
|
|
self.assertEqual(ret[(1, 2, '192.168.2.2')],
|
|
set([(1, 2, '192.168.2.2', 9),
|
|
(1, 2, '192.168.2.2', 10),
|
|
(1, 2, '192.168.2.2', 11)]))
|
|
|
|
def test_is_valid_hostname(self):
|
|
self.assertTrue(is_valid_hostname("local"))
|
|
self.assertTrue(is_valid_hostname("test.test.com"))
|
|
hostname = "test." * 51
|
|
self.assertTrue(is_valid_hostname(hostname))
|
|
hostname = hostname.rstrip('.')
|
|
self.assertTrue(is_valid_hostname(hostname))
|
|
hostname = hostname + "00"
|
|
self.assertFalse(is_valid_hostname(hostname))
|
|
self.assertFalse(is_valid_hostname("$blah#"))
|
|
|
|
def test_is_local_device(self):
|
|
# localhost shows up in whataremyips() output as "::1" for IPv6
|
|
my_ips = ["127.0.0.1", "::1"]
|
|
my_port = 6200
|
|
self.assertTrue(is_local_device(my_ips, my_port,
|
|
"127.0.0.1", my_port))
|
|
self.assertTrue(is_local_device(my_ips, my_port,
|
|
"::1", my_port))
|
|
self.assertTrue(is_local_device(
|
|
my_ips, my_port,
|
|
"0000:0000:0000:0000:0000:0000:0000:0001", my_port))
|
|
self.assertTrue(is_local_device(my_ips, my_port,
|
|
"localhost", my_port))
|
|
self.assertFalse(is_local_device(my_ips, my_port,
|
|
"localhost", my_port + 1))
|
|
self.assertFalse(is_local_device(my_ips, my_port,
|
|
"127.0.0.2", my_port))
|
|
# for those that don't have a local port
|
|
self.assertTrue(is_local_device(my_ips, None,
|
|
my_ips[0], None))
|
|
|
|
# When servers_per_port is active, the "my_port" passed in is None
|
|
# which means "don't include port in the determination of locality
|
|
# because it's not reliable in this deployment scenario"
|
|
self.assertTrue(is_local_device(my_ips, None,
|
|
"127.0.0.1", 6666))
|
|
self.assertTrue(is_local_device(my_ips, None,
|
|
"::1", 6666))
|
|
self.assertTrue(is_local_device(
|
|
my_ips, None,
|
|
"0000:0000:0000:0000:0000:0000:0000:0001", 6666))
|
|
self.assertTrue(is_local_device(my_ips, None,
|
|
"localhost", 6666))
|
|
self.assertFalse(is_local_device(my_ips, None,
|
|
"127.0.0.2", my_port))
|
|
|
|
def test_validate_and_normalize_ip(self):
|
|
ipv4 = "10.0.0.1"
|
|
self.assertEqual(ipv4, validate_and_normalize_ip(ipv4))
|
|
ipv6 = "fe80::204:61ff:fe9d:f156"
|
|
self.assertEqual(ipv6, validate_and_normalize_ip(ipv6.upper()))
|
|
hostname = "test.test.com"
|
|
self.assertRaises(ValueError,
|
|
validate_and_normalize_ip, hostname)
|
|
hostname = "$blah#"
|
|
self.assertRaises(ValueError,
|
|
validate_and_normalize_ip, hostname)
|
|
|
|
def test_validate_and_normalize_address(self):
|
|
ipv4 = "10.0.0.1"
|
|
self.assertEqual(ipv4, validate_and_normalize_address(ipv4))
|
|
ipv6 = "fe80::204:61ff:fe9d:f156"
|
|
self.assertEqual(ipv6, validate_and_normalize_address(ipv6.upper()))
|
|
hostname = "test.test.com"
|
|
self.assertEqual(hostname,
|
|
validate_and_normalize_address(hostname.upper()))
|
|
hostname = "$blah#"
|
|
self.assertRaises(ValueError,
|
|
validate_and_normalize_address, hostname)
|
|
|
|
def test_validate_replicas_by_tier_close(self):
|
|
one_ip_six_devices = \
|
|
defaultdict(float,
|
|
{(): 4.0,
|
|
(0,): 4.0,
|
|
(0, 0): 4.0,
|
|
(0, 0, '127.0.0.1'): 4.0,
|
|
(0, 0, '127.0.0.1', 0): 0.6666666670,
|
|
(0, 0, '127.0.0.1', 1): 0.6666666668,
|
|
(0, 0, '127.0.0.1', 2): 0.6666666667,
|
|
(0, 0, '127.0.0.1', 3): 0.6666666666,
|
|
(0, 0, '127.0.0.1', 4): 0.6666666665,
|
|
(0, 0, '127.0.0.1', 5): 0.6666666664,
|
|
})
|
|
try:
|
|
validate_replicas_by_tier(4, one_ip_six_devices)
|
|
except Exception as e:
|
|
self.fail('one_ip_six_devices is invalid for %s' % e)
|
|
|
|
def test_validate_replicas_by_tier_exact(self):
|
|
three_regions_three_devices = \
|
|
defaultdict(float,
|
|
{(): 3.0,
|
|
(0,): 1.0,
|
|
(0, 0): 1.0,
|
|
(0, 0, '127.0.0.1'): 1.0,
|
|
(0, 0, '127.0.0.1', 0): 1.0,
|
|
(1,): 1.0,
|
|
(1, 1): 1.0,
|
|
(1, 1, '127.0.0.1'): 1.0,
|
|
(1, 1, '127.0.0.1', 1): 1.0,
|
|
(2,): 1.0,
|
|
(2, 2): 1.0,
|
|
(2, 2, '127.0.0.1'): 1.0,
|
|
(2, 2, '127.0.0.1', 2): 1.0,
|
|
})
|
|
try:
|
|
validate_replicas_by_tier(3, three_regions_three_devices)
|
|
except Exception as e:
|
|
self.fail('three_regions_three_devices is invalid for %s' % e)
|
|
|
|
def test_validate_replicas_by_tier_errors(self):
|
|
pseudo_replicas = \
|
|
defaultdict(float,
|
|
{(): 3.0,
|
|
(0,): 1.0,
|
|
(0, 0): 1.0,
|
|
(0, 0, '127.0.0.1'): 1.0,
|
|
(0, 0, '127.0.0.1', 0): 1.0,
|
|
(1,): 1.0,
|
|
(1, 1): 1.0,
|
|
(1, 1, '127.0.0.1'): 1.0,
|
|
(1, 1, '127.0.0.1', 1): 1.0,
|
|
(2,): 1.0,
|
|
(2, 2): 1.0,
|
|
(2, 2, '127.0.0.1'): 1.0,
|
|
(2, 2, '127.0.0.1', 2): 1.0,
|
|
})
|
|
|
|
def do_test(bad_tier_key, bad_tier_name):
|
|
# invalidate a copy of pseudo_replicas at given key and check for
|
|
# an exception to be raised
|
|
test_replicas = dict(pseudo_replicas)
|
|
test_replicas[bad_tier_key] += 0.1 # <- this is not fair!
|
|
with self.assertRaises(exceptions.RingValidationError) as ctx:
|
|
validate_replicas_by_tier(3, test_replicas)
|
|
self.assertEqual(
|
|
'3.1 != 3 at tier %s' % bad_tier_name, str(ctx.exception))
|
|
|
|
do_test((), 'cluster')
|
|
do_test((1,), 'regions')
|
|
do_test((0, 0), 'zones')
|
|
do_test((2, 2, '127.0.0.1'), 'servers')
|
|
do_test((1, 1, '127.0.0.1', 1), 'devices')
|
|
|
|
def test_parse_search_value(self):
|
|
res = parse_search_value('r0')
|
|
self.assertEqual(res, {'region': 0})
|
|
res = parse_search_value('r1')
|
|
self.assertEqual(res, {'region': 1})
|
|
res = parse_search_value('r1z2')
|
|
self.assertEqual(res, {'region': 1, 'zone': 2})
|
|
res = parse_search_value('d1')
|
|
self.assertEqual(res, {'id': 1})
|
|
res = parse_search_value('z1')
|
|
self.assertEqual(res, {'zone': 1})
|
|
res = parse_search_value('-127.0.0.1')
|
|
self.assertEqual(res, {'ip': '127.0.0.1'})
|
|
res = parse_search_value('127.0.0.1')
|
|
self.assertEqual(res, {'ip': '127.0.0.1'})
|
|
res = parse_search_value('-[127.0.0.1]:10001')
|
|
self.assertEqual(res, {'ip': '127.0.0.1', 'port': 10001})
|
|
res = parse_search_value(':10001')
|
|
self.assertEqual(res, {'port': 10001})
|
|
res = parse_search_value('R127.0.0.10')
|
|
self.assertEqual(res, {'replication_ip': '127.0.0.10'})
|
|
res = parse_search_value('R[127.0.0.10]:20000')
|
|
self.assertEqual(res, {'replication_ip': '127.0.0.10',
|
|
'replication_port': 20000})
|
|
res = parse_search_value('R:20000')
|
|
self.assertEqual(res, {'replication_port': 20000})
|
|
res = parse_search_value('/sdb1')
|
|
self.assertEqual(res, {'device': 'sdb1'})
|
|
res = parse_search_value('_meta1')
|
|
self.assertEqual(res, {'meta': 'meta1'})
|
|
self.assertRaises(ValueError, parse_search_value, 'OMGPONIES')
|
|
|
|
def test_parse_search_values_from_opts(self):
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "test.test.com",
|
|
"--port", "6200",
|
|
"--replication-ip", "r.test.com",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "change.test.test.com",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "change.r.test.com",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
expected = {
|
|
'id': 1,
|
|
'region': 2,
|
|
'zone': 3,
|
|
'ip': "test.test.com",
|
|
'port': 6200,
|
|
'replication_ip': "r.test.com",
|
|
'replication_port': 7000,
|
|
'device': "sda3",
|
|
'meta': "some meta data",
|
|
'weight': 3.14159265359,
|
|
}
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
search_values = parse_search_values_from_opts(opts)
|
|
self.assertEqual(search_values, expected)
|
|
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "127.0.0.1",
|
|
"--port", "6200",
|
|
"--replication-ip", "127.0.0.10",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "127.0.0.2",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "127.0.0.20",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
expected = {
|
|
'id': 1,
|
|
'region': 2,
|
|
'zone': 3,
|
|
'ip': "127.0.0.1",
|
|
'port': 6200,
|
|
'replication_ip': "127.0.0.10",
|
|
'replication_port': 7000,
|
|
'device': "sda3",
|
|
'meta': "some meta data",
|
|
'weight': 3.14159265359,
|
|
}
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
search_values = parse_search_values_from_opts(opts)
|
|
self.assertEqual(search_values, expected)
|
|
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "[127.0.0.1]",
|
|
"--port", "6200",
|
|
"--replication-ip", "[127.0.0.10]",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "[127.0.0.2]",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "[127.0.0.20]",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
search_values = parse_search_values_from_opts(opts)
|
|
self.assertEqual(search_values, expected)
|
|
|
|
def test_parse_change_values_from_opts(self):
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "test.test.com",
|
|
"--port", "6200",
|
|
"--replication-ip", "r.test.com",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "change.test.test.com",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "change.r.test.com",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
expected = {
|
|
'ip': "change.test.test.com",
|
|
'port': 6201,
|
|
'replication_ip': "change.r.test.com",
|
|
'replication_port': 7001,
|
|
'device': "sdb3",
|
|
'meta': "some meta data for change",
|
|
}
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
search_values = parse_change_values_from_opts(opts)
|
|
self.assertEqual(search_values, expected)
|
|
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "127.0.0.1",
|
|
"--port", "6200",
|
|
"--replication-ip", "127.0.0.10",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "127.0.0.2",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "127.0.0.20",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
expected = {
|
|
'ip': "127.0.0.2",
|
|
'port': 6201,
|
|
'replication_ip': "127.0.0.20",
|
|
'replication_port': 7001,
|
|
'device': "sdb3",
|
|
'meta': "some meta data for change",
|
|
}
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
search_values = parse_change_values_from_opts(opts)
|
|
self.assertEqual(search_values, expected)
|
|
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "[127.0.0.1]",
|
|
"--port", "6200",
|
|
"--replication-ip", "[127.0.0.10]",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "[127.0.0.2]",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "[127.0.0.20]",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
search_values = parse_change_values_from_opts(opts)
|
|
self.assertEqual(search_values, expected)
|
|
|
|
def test_validate_args(self):
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "test.test.com",
|
|
"--port", "6200",
|
|
"--replication-ip", "r.test.com",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "change.test.test.com",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "change.r.test.com",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertTrue(new_cmd_format)
|
|
self.assertEqual(opts.id, 1)
|
|
self.assertEqual(opts.region, 2)
|
|
self.assertEqual(opts.zone, 3)
|
|
self.assertEqual(opts.ip, "test.test.com")
|
|
self.assertEqual(opts.port, 6200)
|
|
self.assertEqual(opts.replication_ip, "r.test.com")
|
|
self.assertEqual(opts.replication_port, 7000)
|
|
self.assertEqual(opts.device, "sda3")
|
|
self.assertEqual(opts.meta, "some meta data")
|
|
self.assertEqual(opts.weight, 3.14159265359)
|
|
self.assertEqual(opts.change_ip, "change.test.test.com")
|
|
self.assertEqual(opts.change_port, 6201)
|
|
self.assertEqual(opts.change_replication_ip, "change.r.test.com")
|
|
self.assertEqual(opts.change_replication_port, 7001)
|
|
self.assertEqual(opts.change_device, "sdb3")
|
|
self.assertEqual(opts.change_meta, "some meta data for change")
|
|
|
|
def test_validate_args_new_cmd_format(self):
|
|
argv = \
|
|
["--id", "0", "--region", "0", "--zone", "0",
|
|
"--ip", "",
|
|
"--port", "0",
|
|
"--replication-ip", "",
|
|
"--replication-port", "0",
|
|
"--device", "",
|
|
"--meta", "",
|
|
"--weight", "0",
|
|
"--change-ip", "",
|
|
"--change-port", "0",
|
|
"--change-replication-ip", "",
|
|
"--change-replication-port", "0",
|
|
"--change-device", "",
|
|
"--change-meta", ""]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertTrue(new_cmd_format)
|
|
|
|
argv = \
|
|
["--id", None, "--region", None, "--zone", None,
|
|
"--ip", "",
|
|
"--port", "0",
|
|
"--replication-ip", "",
|
|
"--replication-port", "0",
|
|
"--device", "",
|
|
"--meta", "",
|
|
"--weight", None,
|
|
"--change-ip", "change.test.test.com",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "change.r.test.com",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertFalse(new_cmd_format)
|
|
|
|
argv = \
|
|
["--id", "0"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertTrue(new_cmd_format)
|
|
argv = \
|
|
["--region", "0"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertTrue(new_cmd_format)
|
|
argv = \
|
|
["--zone", "0"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertTrue(new_cmd_format)
|
|
argv = \
|
|
["--weight", "0"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertTrue(new_cmd_format)
|
|
|
|
def test_parse_args(self):
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "test.test.com",
|
|
"--port", "6200",
|
|
"--replication-ip", "r.test.com",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "change.test.test.com",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "change.r.test.com",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
|
|
opts, args = parse_args(argv)
|
|
self.assertEqual(opts.id, 1)
|
|
self.assertEqual(opts.region, 2)
|
|
self.assertEqual(opts.zone, 3)
|
|
self.assertEqual(opts.ip, "test.test.com")
|
|
self.assertEqual(opts.port, 6200)
|
|
self.assertEqual(opts.replication_ip, "r.test.com")
|
|
self.assertEqual(opts.replication_port, 7000)
|
|
self.assertEqual(opts.device, "sda3")
|
|
self.assertEqual(opts.meta, "some meta data")
|
|
self.assertEqual(opts.weight, 3.14159265359)
|
|
self.assertEqual(opts.change_ip, "change.test.test.com")
|
|
self.assertEqual(opts.change_port, 6201)
|
|
self.assertEqual(opts.change_replication_ip, "change.r.test.com")
|
|
self.assertEqual(opts.change_replication_port, 7001)
|
|
self.assertEqual(opts.change_device, "sdb3")
|
|
self.assertEqual(opts.change_meta, "some meta data for change")
|
|
self.assertEqual(len(args), 0)
|
|
|
|
def test_parse_builder_ring_filename_args(self):
|
|
args = 'swift-ring-builder object.builder write_ring'
|
|
self.assertEqual((
|
|
'object.builder', 'object.ring.gz'
|
|
), parse_builder_ring_filename_args(args.split()))
|
|
args = 'swift-ring-builder container.ring.gz write_builder'
|
|
self.assertEqual((
|
|
'container.builder', 'container.ring.gz'
|
|
), parse_builder_ring_filename_args(args.split()))
|
|
# builder name arg should always fall through
|
|
args = 'swift-ring-builder test create'
|
|
self.assertEqual((
|
|
'test', 'test.ring.gz'
|
|
), parse_builder_ring_filename_args(args.split()))
|
|
args = 'swift-ring-builder my.file.name create'
|
|
self.assertEqual((
|
|
'my.file.name', 'my.file.name.ring.gz'
|
|
), parse_builder_ring_filename_args(args.split()))
|
|
|
|
def test_build_dev_from_opts(self):
|
|
argv = \
|
|
["--region", "0", "--zone", "3",
|
|
"--ip", "test.test.com",
|
|
"--port", "6200",
|
|
"--replication-ip", "r.test.com",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359"]
|
|
expected = {
|
|
'region': 0,
|
|
'zone': 3,
|
|
'ip': "test.test.com",
|
|
'port': 6200,
|
|
'replication_ip': "r.test.com",
|
|
'replication_port': 7000,
|
|
'device': "sda3",
|
|
'meta': "some meta data",
|
|
'weight': 3.14159265359,
|
|
}
|
|
opts, args = parse_args(argv)
|
|
device = build_dev_from_opts(opts)
|
|
self.assertEqual(device, expected)
|
|
|
|
argv = \
|
|
["--region", "2", "--zone", "3",
|
|
"--ip", "[test.test.com]",
|
|
"--port", "6200",
|
|
"--replication-ip", "[r.test.com]",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359"]
|
|
opts, args = parse_args(argv)
|
|
self.assertRaises(ValueError, build_dev_from_opts, opts)
|
|
|
|
argv = \
|
|
["--region", "2", "--zone", "3",
|
|
"--ip", "[test.test.com]",
|
|
"--port", "6200",
|
|
"--replication-ip", "[r.test.com]",
|
|
"--replication-port", "7000",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359"]
|
|
opts, args = parse_args(argv)
|
|
self.assertRaises(ValueError, build_dev_from_opts, opts)
|
|
|
|
def test_replication_defaults(self):
|
|
args = '-r 1 -z 1 -i 127.0.0.1 -p 6010 -d d1 -w 100'.split()
|
|
opts, _ = parse_args(args)
|
|
device = build_dev_from_opts(opts)
|
|
expected = {
|
|
'device': 'd1',
|
|
'ip': '127.0.0.1',
|
|
'meta': '',
|
|
'port': 6010,
|
|
'region': 1,
|
|
'replication_ip': '127.0.0.1',
|
|
'replication_port': 6010,
|
|
'weight': 100.0,
|
|
'zone': 1,
|
|
}
|
|
self.assertEqual(device, expected)
|
|
|
|
args = '-r 1 -z 1 -i test.com -p 6010 -d d1 -w 100'.split()
|
|
opts, _ = parse_args(args)
|
|
device = build_dev_from_opts(opts)
|
|
expected = {
|
|
'device': 'd1',
|
|
'ip': 'test.com',
|
|
'meta': '',
|
|
'port': 6010,
|
|
'region': 1,
|
|
'replication_ip': 'test.com',
|
|
'replication_port': 6010,
|
|
'weight': 100.0,
|
|
'zone': 1,
|
|
}
|
|
self.assertEqual(device, expected)
|
|
|
|
@unittest.skipIf(sys.version_info >= (3,),
|
|
"Seed-specific tests don't work well on py3")
|
|
def test_dispersion_report(self):
|
|
rb = ring.RingBuilder(8, 3, 0)
|
|
rb.add_dev({'id': 0, 'region': 1, 'zone': 0, 'weight': 100,
|
|
'ip': '127.0.0.0', 'port': 10000, 'device': 'sda1'})
|
|
rb.add_dev({'id': 3, 'region': 1, 'zone': 0, 'weight': 100,
|
|
'ip': '127.0.0.0', 'port': 10000, 'device': 'sdb1'})
|
|
rb.add_dev({'id': 4, 'region': 1, 'zone': 0, 'weight': 100,
|
|
'ip': '127.0.0.0', 'port': 10000, 'device': 'sdc1'})
|
|
rb.add_dev({'id': 5, 'region': 1, 'zone': 0, 'weight': 100,
|
|
'ip': '127.0.0.0', 'port': 10000, 'device': 'sdd1'})
|
|
|
|
rb.add_dev({'id': 1, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
|
|
rb.add_dev({'id': 6, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.1', 'port': 10001, 'device': 'sdb1'})
|
|
rb.add_dev({'id': 7, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.1', 'port': 10001, 'device': 'sdc1'})
|
|
rb.add_dev({'id': 8, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.1', 'port': 10001, 'device': 'sdd1'})
|
|
|
|
rb.add_dev({'id': 2, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.2', 'port': 10002, 'device': 'sda1'})
|
|
rb.add_dev({'id': 9, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.2', 'port': 10002, 'device': 'sdb1'})
|
|
rb.add_dev({'id': 10, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.2', 'port': 10002, 'device': 'sdc1'})
|
|
rb.add_dev({'id': 11, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.2', 'port': 10002, 'device': 'sdd1'})
|
|
|
|
# this ring is pretty volatile and the assertions are pretty brittle
|
|
# so we use a specific seed
|
|
rb.rebalance(seed=100)
|
|
rb.validate()
|
|
|
|
self.assertEqual(rb.dispersion, 18.489583333333332)
|
|
report = dispersion_report(rb)
|
|
self.assertEqual(report['worst_tier'], 'r1z1-127.0.0.1')
|
|
self.assertEqual(report['max_dispersion'], 22.68370607028754)
|
|
|
|
def build_tier_report(max_replicas, placed_parts, dispersion,
|
|
replicas):
|
|
return {
|
|
'max_replicas': max_replicas,
|
|
'placed_parts': placed_parts,
|
|
'dispersion': dispersion,
|
|
'replicas': replicas,
|
|
}
|
|
|
|
# every partition has at least two replicas in this zone, unfortunately
|
|
# sometimes they're both on the same server.
|
|
expected = [
|
|
['r1z1', build_tier_report(
|
|
2, 627, 18.341307814992025, [0, 0, 141, 115])],
|
|
['r1z1-127.0.0.1', build_tier_report(
|
|
1, 313, 22.68370607028754, [14, 171, 71, 0])],
|
|
['r1z1-127.0.0.2', build_tier_report(
|
|
1, 314, 22.611464968152866, [13, 172, 71, 0])],
|
|
]
|
|
report = dispersion_report(rb, 'r1z1[^/]*$', verbose=True)
|
|
graph = report['graph']
|
|
for i, (expected_key, expected_report) in enumerate(expected):
|
|
key, report = graph[i]
|
|
self.assertEqual(
|
|
(key, report),
|
|
(expected_key, expected_report)
|
|
)
|
|
|
|
# overcompensate in r1z0
|
|
rb.add_dev({'id': 12, 'region': 1, 'zone': 0, 'weight': 500,
|
|
'ip': '127.0.0.3', 'port': 10003, 'device': 'sda1'})
|
|
rb.add_dev({'id': 13, 'region': 1, 'zone': 0, 'weight': 500,
|
|
'ip': '127.0.0.3', 'port': 10003, 'device': 'sdb1'})
|
|
rb.add_dev({'id': 14, 'region': 1, 'zone': 0, 'weight': 500,
|
|
'ip': '127.0.0.3', 'port': 10003, 'device': 'sdc1'})
|
|
rb.add_dev({'id': 15, 'region': 1, 'zone': 0, 'weight': 500,
|
|
'ip': '127.0.0.3', 'port': 10003, 'device': 'sdd1'})
|
|
|
|
# when the biggest tier has the smallest devices things get ugly
|
|
# can't move all the part-replicas in one rebalance
|
|
rb.rebalance(seed=100)
|
|
report = dispersion_report(rb, verbose=True)
|
|
self.assertEqual(rb.dispersion, 3.90625)
|
|
self.assertEqual(report['worst_tier'], 'r1z1-127.0.0.2')
|
|
self.assertEqual(report['max_dispersion'], 8.152173913043478)
|
|
# do a sencond rebalance
|
|
rb.rebalance(seed=100)
|
|
report = dispersion_report(rb, verbose=True)
|
|
self.assertEqual(rb.dispersion, 16.666666666666668)
|
|
self.assertEqual(report['worst_tier'], 'r1z0-127.0.0.3')
|
|
self.assertEqual(report['max_dispersion'], 33.333333333333336)
|
|
|
|
# ... but overload can square it
|
|
rb.set_overload(rb.get_required_overload())
|
|
rb.rebalance()
|
|
self.assertEqual(rb.dispersion, 0.0)
|
|
|
|
def test_parse_address_old_format(self):
|
|
# Test old format
|
|
argv = "127.0.0.1:6200R127.0.0.1:6200/sda1_some meta data"
|
|
ip, port, rest = parse_address(argv)
|
|
self.assertEqual(ip, '127.0.0.1')
|
|
self.assertEqual(port, 6200)
|
|
self.assertEqual(rest, 'R127.0.0.1:6200/sda1_some meta data')
|
|
|
|
def test_normalized_device_tier_names(self):
|
|
rb = ring.RingBuilder(8, 3, 0)
|
|
rb.add_dev({
|
|
'region': 1,
|
|
'zone': 1,
|
|
'ip': '127.0.0.1',
|
|
'port': 6011,
|
|
'device': 'd1',
|
|
'weight': 0.0,
|
|
})
|
|
dev = rb.devs[0]
|
|
expected = 'r1z1-127.0.0.1/d1'
|
|
self.assertEqual(expected, get_tier_name(tiers_for_dev(dev)[-1], rb))
|
|
self.assertEqual(expected, pretty_dev(dev))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|