7013e70ca6
With a sufficiently undispersed ring it's possible to move an entire replicas worth of parts and yet the value of dispersion may not get any better (even though in reality dispersion has dramatically improved). The problem is dispersion will currently only represent up to one whole replica worth of parts being undispersed. However with EC rings it's possible for more than one whole replicas worth of partitions to be undispersed, in these cases the builder will require multiple rebalance operations to fully disperse replicas - but the dispersion value should improve with every rebalance. N.B. with this change it's possible for rings with a bad dispersion value to measure as having a significantly smaller dispersion value after a rebalance (even though they may not have had their dispersion change) because the total amount of bad dispersion we can measure has been increased but we're normalizing within a similar range. Closes-Bug: #1697543 Change-Id: Ifefff0260deac0c3e8b369a1e158686c89936686
710 lines
30 KiB
Python
710 lines
30 KiB
Python
# Copyright (c) 2010-2012 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import unittest
|
|
|
|
from swift.common import ring
|
|
from swift.common.ring.utils import (tiers_for_dev, build_tier_tree,
|
|
validate_and_normalize_ip,
|
|
validate_and_normalize_address,
|
|
is_valid_hostname,
|
|
is_local_device, parse_search_value,
|
|
parse_search_values_from_opts,
|
|
parse_change_values_from_opts,
|
|
validate_args, parse_args,
|
|
parse_builder_ring_filename_args,
|
|
build_dev_from_opts, dispersion_report,
|
|
parse_address, get_tier_name, pretty_dev)
|
|
|
|
|
|
class TestUtils(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.test_dev = {'region': 1, 'zone': 1, 'ip': '192.168.1.1',
|
|
'port': '6200', 'id': 0}
|
|
|
|
def get_test_devs():
|
|
dev0 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1',
|
|
'port': '6200', 'id': 0}
|
|
dev1 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1',
|
|
'port': '6200', 'id': 1}
|
|
dev2 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1',
|
|
'port': '6200', 'id': 2}
|
|
dev3 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2',
|
|
'port': '6200', 'id': 3}
|
|
dev4 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2',
|
|
'port': '6200', 'id': 4}
|
|
dev5 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2',
|
|
'port': '6200', 'id': 5}
|
|
dev6 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1',
|
|
'port': '6200', 'id': 6}
|
|
dev7 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1',
|
|
'port': '6200', 'id': 7}
|
|
dev8 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1',
|
|
'port': '6200', 'id': 8}
|
|
dev9 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2',
|
|
'port': '6200', 'id': 9}
|
|
dev10 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2',
|
|
'port': '6200', 'id': 10}
|
|
dev11 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2',
|
|
'port': '6200', 'id': 11}
|
|
return [dev0, dev1, dev2, dev3, dev4, dev5,
|
|
dev6, dev7, dev8, dev9, dev10, dev11]
|
|
|
|
self.test_devs = get_test_devs()
|
|
|
|
def test_tiers_for_dev(self):
|
|
self.assertEqual(
|
|
tiers_for_dev(self.test_dev),
|
|
((1,),
|
|
(1, 1),
|
|
(1, 1, '192.168.1.1'),
|
|
(1, 1, '192.168.1.1', 0)))
|
|
|
|
def test_build_tier_tree(self):
|
|
ret = build_tier_tree(self.test_devs)
|
|
self.assertEqual(len(ret), 8)
|
|
self.assertEqual(ret[()], set([(1,)]))
|
|
self.assertEqual(ret[(1,)], set([(1, 1), (1, 2)]))
|
|
self.assertEqual(ret[(1, 1)],
|
|
set([(1, 1, '192.168.1.2'),
|
|
(1, 1, '192.168.1.1')]))
|
|
self.assertEqual(ret[(1, 2)],
|
|
set([(1, 2, '192.168.2.2'),
|
|
(1, 2, '192.168.2.1')]))
|
|
self.assertEqual(ret[(1, 1, '192.168.1.1')],
|
|
set([(1, 1, '192.168.1.1', 0),
|
|
(1, 1, '192.168.1.1', 1),
|
|
(1, 1, '192.168.1.1', 2)]))
|
|
self.assertEqual(ret[(1, 1, '192.168.1.2')],
|
|
set([(1, 1, '192.168.1.2', 3),
|
|
(1, 1, '192.168.1.2', 4),
|
|
(1, 1, '192.168.1.2', 5)]))
|
|
self.assertEqual(ret[(1, 2, '192.168.2.1')],
|
|
set([(1, 2, '192.168.2.1', 6),
|
|
(1, 2, '192.168.2.1', 7),
|
|
(1, 2, '192.168.2.1', 8)]))
|
|
self.assertEqual(ret[(1, 2, '192.168.2.2')],
|
|
set([(1, 2, '192.168.2.2', 9),
|
|
(1, 2, '192.168.2.2', 10),
|
|
(1, 2, '192.168.2.2', 11)]))
|
|
|
|
def test_is_valid_hostname(self):
|
|
self.assertTrue(is_valid_hostname("local"))
|
|
self.assertTrue(is_valid_hostname("test.test.com"))
|
|
hostname = "test." * 51
|
|
self.assertTrue(is_valid_hostname(hostname))
|
|
hostname = hostname.rstrip('.')
|
|
self.assertTrue(is_valid_hostname(hostname))
|
|
hostname = hostname + "00"
|
|
self.assertFalse(is_valid_hostname(hostname))
|
|
self.assertFalse(is_valid_hostname("$blah#"))
|
|
|
|
def test_is_local_device(self):
|
|
# localhost shows up in whataremyips() output as "::1" for IPv6
|
|
my_ips = ["127.0.0.1", "::1"]
|
|
my_port = 6200
|
|
self.assertTrue(is_local_device(my_ips, my_port,
|
|
"127.0.0.1", my_port))
|
|
self.assertTrue(is_local_device(my_ips, my_port,
|
|
"::1", my_port))
|
|
self.assertTrue(is_local_device(
|
|
my_ips, my_port,
|
|
"0000:0000:0000:0000:0000:0000:0000:0001", my_port))
|
|
self.assertTrue(is_local_device(my_ips, my_port,
|
|
"localhost", my_port))
|
|
self.assertFalse(is_local_device(my_ips, my_port,
|
|
"localhost", my_port + 1))
|
|
self.assertFalse(is_local_device(my_ips, my_port,
|
|
"127.0.0.2", my_port))
|
|
# for those that don't have a local port
|
|
self.assertTrue(is_local_device(my_ips, None,
|
|
my_ips[0], None))
|
|
|
|
# When servers_per_port is active, the "my_port" passed in is None
|
|
# which means "don't include port in the determination of locality
|
|
# because it's not reliable in this deployment scenario"
|
|
self.assertTrue(is_local_device(my_ips, None,
|
|
"127.0.0.1", 6666))
|
|
self.assertTrue(is_local_device(my_ips, None,
|
|
"::1", 6666))
|
|
self.assertTrue(is_local_device(
|
|
my_ips, None,
|
|
"0000:0000:0000:0000:0000:0000:0000:0001", 6666))
|
|
self.assertTrue(is_local_device(my_ips, None,
|
|
"localhost", 6666))
|
|
self.assertFalse(is_local_device(my_ips, None,
|
|
"127.0.0.2", my_port))
|
|
|
|
def test_validate_and_normalize_ip(self):
|
|
ipv4 = "10.0.0.1"
|
|
self.assertEqual(ipv4, validate_and_normalize_ip(ipv4))
|
|
ipv6 = "fe80::204:61ff:fe9d:f156"
|
|
self.assertEqual(ipv6, validate_and_normalize_ip(ipv6.upper()))
|
|
hostname = "test.test.com"
|
|
self.assertRaises(ValueError,
|
|
validate_and_normalize_ip, hostname)
|
|
hostname = "$blah#"
|
|
self.assertRaises(ValueError,
|
|
validate_and_normalize_ip, hostname)
|
|
|
|
def test_validate_and_normalize_address(self):
|
|
ipv4 = "10.0.0.1"
|
|
self.assertEqual(ipv4, validate_and_normalize_address(ipv4))
|
|
ipv6 = "fe80::204:61ff:fe9d:f156"
|
|
self.assertEqual(ipv6, validate_and_normalize_address(ipv6.upper()))
|
|
hostname = "test.test.com"
|
|
self.assertEqual(hostname,
|
|
validate_and_normalize_address(hostname.upper()))
|
|
hostname = "$blah#"
|
|
self.assertRaises(ValueError,
|
|
validate_and_normalize_address, hostname)
|
|
|
|
def test_parse_search_value(self):
|
|
res = parse_search_value('r0')
|
|
self.assertEqual(res, {'region': 0})
|
|
res = parse_search_value('r1')
|
|
self.assertEqual(res, {'region': 1})
|
|
res = parse_search_value('r1z2')
|
|
self.assertEqual(res, {'region': 1, 'zone': 2})
|
|
res = parse_search_value('d1')
|
|
self.assertEqual(res, {'id': 1})
|
|
res = parse_search_value('z1')
|
|
self.assertEqual(res, {'zone': 1})
|
|
res = parse_search_value('-127.0.0.1')
|
|
self.assertEqual(res, {'ip': '127.0.0.1'})
|
|
res = parse_search_value('127.0.0.1')
|
|
self.assertEqual(res, {'ip': '127.0.0.1'})
|
|
res = parse_search_value('-[127.0.0.1]:10001')
|
|
self.assertEqual(res, {'ip': '127.0.0.1', 'port': 10001})
|
|
res = parse_search_value(':10001')
|
|
self.assertEqual(res, {'port': 10001})
|
|
res = parse_search_value('R127.0.0.10')
|
|
self.assertEqual(res, {'replication_ip': '127.0.0.10'})
|
|
res = parse_search_value('R[127.0.0.10]:20000')
|
|
self.assertEqual(res, {'replication_ip': '127.0.0.10',
|
|
'replication_port': 20000})
|
|
res = parse_search_value('R:20000')
|
|
self.assertEqual(res, {'replication_port': 20000})
|
|
res = parse_search_value('/sdb1')
|
|
self.assertEqual(res, {'device': 'sdb1'})
|
|
res = parse_search_value('_meta1')
|
|
self.assertEqual(res, {'meta': 'meta1'})
|
|
self.assertRaises(ValueError, parse_search_value, 'OMGPONIES')
|
|
|
|
def test_parse_search_values_from_opts(self):
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "test.test.com",
|
|
"--port", "6200",
|
|
"--replication-ip", "r.test.com",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "change.test.test.com",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "change.r.test.com",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
expected = {
|
|
'id': 1,
|
|
'region': 2,
|
|
'zone': 3,
|
|
'ip': "test.test.com",
|
|
'port': 6200,
|
|
'replication_ip': "r.test.com",
|
|
'replication_port': 7000,
|
|
'device': "sda3",
|
|
'meta': "some meta data",
|
|
'weight': 3.14159265359,
|
|
}
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
search_values = parse_search_values_from_opts(opts)
|
|
self.assertEqual(search_values, expected)
|
|
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "127.0.0.1",
|
|
"--port", "6200",
|
|
"--replication-ip", "127.0.0.10",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "127.0.0.2",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "127.0.0.20",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
expected = {
|
|
'id': 1,
|
|
'region': 2,
|
|
'zone': 3,
|
|
'ip': "127.0.0.1",
|
|
'port': 6200,
|
|
'replication_ip': "127.0.0.10",
|
|
'replication_port': 7000,
|
|
'device': "sda3",
|
|
'meta': "some meta data",
|
|
'weight': 3.14159265359,
|
|
}
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
search_values = parse_search_values_from_opts(opts)
|
|
self.assertEqual(search_values, expected)
|
|
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "[127.0.0.1]",
|
|
"--port", "6200",
|
|
"--replication-ip", "[127.0.0.10]",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "[127.0.0.2]",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "[127.0.0.20]",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
search_values = parse_search_values_from_opts(opts)
|
|
self.assertEqual(search_values, expected)
|
|
|
|
def test_parse_change_values_from_opts(self):
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "test.test.com",
|
|
"--port", "6200",
|
|
"--replication-ip", "r.test.com",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "change.test.test.com",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "change.r.test.com",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
expected = {
|
|
'ip': "change.test.test.com",
|
|
'port': 6201,
|
|
'replication_ip': "change.r.test.com",
|
|
'replication_port': 7001,
|
|
'device': "sdb3",
|
|
'meta': "some meta data for change",
|
|
}
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
search_values = parse_change_values_from_opts(opts)
|
|
self.assertEqual(search_values, expected)
|
|
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "127.0.0.1",
|
|
"--port", "6200",
|
|
"--replication-ip", "127.0.0.10",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "127.0.0.2",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "127.0.0.20",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
expected = {
|
|
'ip': "127.0.0.2",
|
|
'port': 6201,
|
|
'replication_ip': "127.0.0.20",
|
|
'replication_port': 7001,
|
|
'device': "sdb3",
|
|
'meta': "some meta data for change",
|
|
}
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
search_values = parse_change_values_from_opts(opts)
|
|
self.assertEqual(search_values, expected)
|
|
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "[127.0.0.1]",
|
|
"--port", "6200",
|
|
"--replication-ip", "[127.0.0.10]",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "[127.0.0.2]",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "[127.0.0.20]",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
search_values = parse_change_values_from_opts(opts)
|
|
self.assertEqual(search_values, expected)
|
|
|
|
def test_validate_args(self):
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "test.test.com",
|
|
"--port", "6200",
|
|
"--replication-ip", "r.test.com",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "change.test.test.com",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "change.r.test.com",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertTrue(new_cmd_format)
|
|
self.assertEqual(opts.id, 1)
|
|
self.assertEqual(opts.region, 2)
|
|
self.assertEqual(opts.zone, 3)
|
|
self.assertEqual(opts.ip, "test.test.com")
|
|
self.assertEqual(opts.port, 6200)
|
|
self.assertEqual(opts.replication_ip, "r.test.com")
|
|
self.assertEqual(opts.replication_port, 7000)
|
|
self.assertEqual(opts.device, "sda3")
|
|
self.assertEqual(opts.meta, "some meta data")
|
|
self.assertEqual(opts.weight, 3.14159265359)
|
|
self.assertEqual(opts.change_ip, "change.test.test.com")
|
|
self.assertEqual(opts.change_port, 6201)
|
|
self.assertEqual(opts.change_replication_ip, "change.r.test.com")
|
|
self.assertEqual(opts.change_replication_port, 7001)
|
|
self.assertEqual(opts.change_device, "sdb3")
|
|
self.assertEqual(opts.change_meta, "some meta data for change")
|
|
|
|
def test_validate_args_new_cmd_format(self):
|
|
argv = \
|
|
["--id", "0", "--region", "0", "--zone", "0",
|
|
"--ip", "",
|
|
"--port", "0",
|
|
"--replication-ip", "",
|
|
"--replication-port", "0",
|
|
"--device", "",
|
|
"--meta", "",
|
|
"--weight", "0",
|
|
"--change-ip", "",
|
|
"--change-port", "0",
|
|
"--change-replication-ip", "",
|
|
"--change-replication-port", "0",
|
|
"--change-device", "",
|
|
"--change-meta", ""]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertTrue(new_cmd_format)
|
|
|
|
argv = \
|
|
["--id", None, "--region", None, "--zone", None,
|
|
"--ip", "",
|
|
"--port", "0",
|
|
"--replication-ip", "",
|
|
"--replication-port", "0",
|
|
"--device", "",
|
|
"--meta", "",
|
|
"--weight", None,
|
|
"--change-ip", "change.test.test.com",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "change.r.test.com",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertFalse(new_cmd_format)
|
|
|
|
argv = \
|
|
["--id", "0"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertTrue(new_cmd_format)
|
|
argv = \
|
|
["--region", "0"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertTrue(new_cmd_format)
|
|
argv = \
|
|
["--zone", "0"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertTrue(new_cmd_format)
|
|
argv = \
|
|
["--weight", "0"]
|
|
new_cmd_format, opts, args = validate_args(argv)
|
|
self.assertTrue(new_cmd_format)
|
|
|
|
def test_parse_args(self):
|
|
argv = \
|
|
["--id", "1", "--region", "2", "--zone", "3",
|
|
"--ip", "test.test.com",
|
|
"--port", "6200",
|
|
"--replication-ip", "r.test.com",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359",
|
|
"--change-ip", "change.test.test.com",
|
|
"--change-port", "6201",
|
|
"--change-replication-ip", "change.r.test.com",
|
|
"--change-replication-port", "7001",
|
|
"--change-device", "sdb3",
|
|
"--change-meta", "some meta data for change"]
|
|
|
|
opts, args = parse_args(argv)
|
|
self.assertEqual(opts.id, 1)
|
|
self.assertEqual(opts.region, 2)
|
|
self.assertEqual(opts.zone, 3)
|
|
self.assertEqual(opts.ip, "test.test.com")
|
|
self.assertEqual(opts.port, 6200)
|
|
self.assertEqual(opts.replication_ip, "r.test.com")
|
|
self.assertEqual(opts.replication_port, 7000)
|
|
self.assertEqual(opts.device, "sda3")
|
|
self.assertEqual(opts.meta, "some meta data")
|
|
self.assertEqual(opts.weight, 3.14159265359)
|
|
self.assertEqual(opts.change_ip, "change.test.test.com")
|
|
self.assertEqual(opts.change_port, 6201)
|
|
self.assertEqual(opts.change_replication_ip, "change.r.test.com")
|
|
self.assertEqual(opts.change_replication_port, 7001)
|
|
self.assertEqual(opts.change_device, "sdb3")
|
|
self.assertEqual(opts.change_meta, "some meta data for change")
|
|
self.assertEqual(len(args), 0)
|
|
|
|
def test_parse_builder_ring_filename_args(self):
|
|
args = 'swift-ring-builder object.builder write_ring'
|
|
self.assertEqual((
|
|
'object.builder', 'object.ring.gz'
|
|
), parse_builder_ring_filename_args(args.split()))
|
|
args = 'swift-ring-builder container.ring.gz write_builder'
|
|
self.assertEqual((
|
|
'container.builder', 'container.ring.gz'
|
|
), parse_builder_ring_filename_args(args.split()))
|
|
# builder name arg should always fall through
|
|
args = 'swift-ring-builder test create'
|
|
self.assertEqual((
|
|
'test', 'test.ring.gz'
|
|
), parse_builder_ring_filename_args(args.split()))
|
|
args = 'swift-ring-builder my.file.name create'
|
|
self.assertEqual((
|
|
'my.file.name', 'my.file.name.ring.gz'
|
|
), parse_builder_ring_filename_args(args.split()))
|
|
|
|
def test_build_dev_from_opts(self):
|
|
argv = \
|
|
["--region", "0", "--zone", "3",
|
|
"--ip", "test.test.com",
|
|
"--port", "6200",
|
|
"--replication-ip", "r.test.com",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359"]
|
|
expected = {
|
|
'region': 0,
|
|
'zone': 3,
|
|
'ip': "test.test.com",
|
|
'port': 6200,
|
|
'replication_ip': "r.test.com",
|
|
'replication_port': 7000,
|
|
'device': "sda3",
|
|
'meta': "some meta data",
|
|
'weight': 3.14159265359,
|
|
}
|
|
opts, args = parse_args(argv)
|
|
device = build_dev_from_opts(opts)
|
|
self.assertEqual(device, expected)
|
|
|
|
argv = \
|
|
["--region", "2", "--zone", "3",
|
|
"--ip", "[test.test.com]",
|
|
"--port", "6200",
|
|
"--replication-ip", "[r.test.com]",
|
|
"--replication-port", "7000",
|
|
"--device", "sda3",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359"]
|
|
opts, args = parse_args(argv)
|
|
self.assertRaises(ValueError, build_dev_from_opts, opts)
|
|
|
|
argv = \
|
|
["--region", "2", "--zone", "3",
|
|
"--ip", "[test.test.com]",
|
|
"--port", "6200",
|
|
"--replication-ip", "[r.test.com]",
|
|
"--replication-port", "7000",
|
|
"--meta", "some meta data",
|
|
"--weight", "3.14159265359"]
|
|
opts, args = parse_args(argv)
|
|
self.assertRaises(ValueError, build_dev_from_opts, opts)
|
|
|
|
def test_replication_defaults(self):
|
|
args = '-r 1 -z 1 -i 127.0.0.1 -p 6010 -d d1 -w 100'.split()
|
|
opts, _ = parse_args(args)
|
|
device = build_dev_from_opts(opts)
|
|
expected = {
|
|
'device': 'd1',
|
|
'ip': '127.0.0.1',
|
|
'meta': '',
|
|
'port': 6010,
|
|
'region': 1,
|
|
'replication_ip': '127.0.0.1',
|
|
'replication_port': 6010,
|
|
'weight': 100.0,
|
|
'zone': 1,
|
|
}
|
|
self.assertEqual(device, expected)
|
|
|
|
args = '-r 1 -z 1 -i test.com -p 6010 -d d1 -w 100'.split()
|
|
opts, _ = parse_args(args)
|
|
device = build_dev_from_opts(opts)
|
|
expected = {
|
|
'device': 'd1',
|
|
'ip': 'test.com',
|
|
'meta': '',
|
|
'port': 6010,
|
|
'region': 1,
|
|
'replication_ip': 'test.com',
|
|
'replication_port': 6010,
|
|
'weight': 100.0,
|
|
'zone': 1,
|
|
}
|
|
self.assertEqual(device, expected)
|
|
|
|
def test_dispersion_report(self):
|
|
rb = ring.RingBuilder(8, 3, 0)
|
|
rb.add_dev({'id': 0, 'region': 1, 'zone': 0, 'weight': 100,
|
|
'ip': '127.0.0.0', 'port': 10000, 'device': 'sda1'})
|
|
rb.add_dev({'id': 3, 'region': 1, 'zone': 0, 'weight': 100,
|
|
'ip': '127.0.0.0', 'port': 10000, 'device': 'sdb1'})
|
|
rb.add_dev({'id': 4, 'region': 1, 'zone': 0, 'weight': 100,
|
|
'ip': '127.0.0.0', 'port': 10000, 'device': 'sdc1'})
|
|
rb.add_dev({'id': 5, 'region': 1, 'zone': 0, 'weight': 100,
|
|
'ip': '127.0.0.0', 'port': 10000, 'device': 'sdd1'})
|
|
|
|
rb.add_dev({'id': 1, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
|
|
rb.add_dev({'id': 6, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.1', 'port': 10001, 'device': 'sdb1'})
|
|
rb.add_dev({'id': 7, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.1', 'port': 10001, 'device': 'sdc1'})
|
|
rb.add_dev({'id': 8, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.1', 'port': 10001, 'device': 'sdd1'})
|
|
|
|
rb.add_dev({'id': 2, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.2', 'port': 10002, 'device': 'sda1'})
|
|
rb.add_dev({'id': 9, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.2', 'port': 10002, 'device': 'sdb1'})
|
|
rb.add_dev({'id': 10, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.2', 'port': 10002, 'device': 'sdc1'})
|
|
rb.add_dev({'id': 11, 'region': 1, 'zone': 1, 'weight': 200,
|
|
'ip': '127.0.0.2', 'port': 10002, 'device': 'sdd1'})
|
|
|
|
# this ring is pretty volatile and the assertions are pretty brittle
|
|
# so we use a specific seed
|
|
rb.rebalance(seed=100)
|
|
rb.validate()
|
|
|
|
self.assertEqual(rb.dispersion, 18.489583333333332)
|
|
report = dispersion_report(rb)
|
|
self.assertEqual(report['worst_tier'], 'r1z1-127.0.0.1')
|
|
self.assertEqual(report['max_dispersion'], 22.68370607028754)
|
|
|
|
def build_tier_report(max_replicas, placed_parts, dispersion,
|
|
replicas):
|
|
return {
|
|
'max_replicas': max_replicas,
|
|
'placed_parts': placed_parts,
|
|
'dispersion': dispersion,
|
|
'replicas': replicas,
|
|
}
|
|
|
|
# every partition has at least two replicas in this zone, unfortunately
|
|
# sometimes they're both on the same server.
|
|
expected = [
|
|
['r1z1', build_tier_report(
|
|
2, 627, 18.341307814992025, [0, 0, 141, 115])],
|
|
['r1z1-127.0.0.1', build_tier_report(
|
|
1, 313, 22.68370607028754, [14, 171, 71, 0])],
|
|
['r1z1-127.0.0.2', build_tier_report(
|
|
1, 314, 22.611464968152866, [13, 172, 71, 0])],
|
|
]
|
|
report = dispersion_report(rb, 'r1z1[^/]*$', verbose=True)
|
|
graph = report['graph']
|
|
for i, (expected_key, expected_report) in enumerate(expected):
|
|
key, report = graph[i]
|
|
self.assertEqual(
|
|
(key, report),
|
|
(expected_key, expected_report)
|
|
)
|
|
|
|
# overcompensate in r1z0
|
|
rb.add_dev({'id': 12, 'region': 1, 'zone': 0, 'weight': 500,
|
|
'ip': '127.0.0.3', 'port': 10003, 'device': 'sda1'})
|
|
rb.add_dev({'id': 13, 'region': 1, 'zone': 0, 'weight': 500,
|
|
'ip': '127.0.0.3', 'port': 10003, 'device': 'sdb1'})
|
|
rb.add_dev({'id': 14, 'region': 1, 'zone': 0, 'weight': 500,
|
|
'ip': '127.0.0.3', 'port': 10003, 'device': 'sdc1'})
|
|
rb.add_dev({'id': 15, 'region': 1, 'zone': 0, 'weight': 500,
|
|
'ip': '127.0.0.3', 'port': 10003, 'device': 'sdd1'})
|
|
|
|
# when the biggest tier has the smallest devices things get ugly
|
|
# can't move all the part-replicas in one rebalance
|
|
rb.rebalance(seed=100)
|
|
report = dispersion_report(rb, verbose=True)
|
|
self.assertEqual(rb.dispersion, 3.90625)
|
|
self.assertEqual(report['worst_tier'], 'r1z1-127.0.0.2')
|
|
self.assertEqual(report['max_dispersion'], 8.152173913043478)
|
|
# do a sencond rebalance
|
|
rb.rebalance(seed=100)
|
|
report = dispersion_report(rb, verbose=True)
|
|
self.assertEqual(rb.dispersion, 16.666666666666668)
|
|
self.assertEqual(report['worst_tier'], 'r1z0-127.0.0.3')
|
|
self.assertEqual(report['max_dispersion'], 33.333333333333336)
|
|
|
|
# ... but overload can square it
|
|
rb.set_overload(rb.get_required_overload())
|
|
rb.rebalance()
|
|
self.assertEqual(rb.dispersion, 0.0)
|
|
|
|
def test_parse_address_old_format(self):
|
|
# Test old format
|
|
argv = "127.0.0.1:6200R127.0.0.1:6200/sda1_some meta data"
|
|
ip, port, rest = parse_address(argv)
|
|
self.assertEqual(ip, '127.0.0.1')
|
|
self.assertEqual(port, 6200)
|
|
self.assertEqual(rest, 'R127.0.0.1:6200/sda1_some meta data')
|
|
|
|
def test_normalized_device_tier_names(self):
|
|
rb = ring.RingBuilder(8, 3, 0)
|
|
rb.add_dev({
|
|
'region': 1,
|
|
'zone': 1,
|
|
'ip': '127.0.0.1',
|
|
'port': 6011,
|
|
'device': 'd1',
|
|
'weight': 0.0,
|
|
})
|
|
dev = rb.devs[0]
|
|
expected = 'r1z1-127.0.0.1/d1'
|
|
self.assertEqual(expected, get_tier_name(tiers_for_dev(dev)[-1], rb))
|
|
self.assertEqual(expected, pretty_dev(dev))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|