1310 lines
56 KiB
Python
1310 lines
56 KiB
Python
# Copyright (c) 2010-2017 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
import json
|
|
from contextlib import contextmanager
|
|
|
|
import mock
|
|
import os
|
|
import random
|
|
import tempfile
|
|
import unittest
|
|
import shutil
|
|
import copy
|
|
import time
|
|
|
|
from collections import defaultdict, Counter
|
|
|
|
from swift.common.exceptions import RingBuilderError
|
|
from swift.common.ring import RingBuilder, Ring
|
|
from swift.common.ring.composite_builder import (
|
|
compose_rings, CompositeRingBuilder, CooperativeRingBuilder)
|
|
|
|
|
|
def make_device_iter():
|
|
x = 0
|
|
base_port = 6000
|
|
while True:
|
|
yield {'region': 0, # Note that region may be replaced on the tests
|
|
'zone': 0,
|
|
'ip': '10.0.0.%s' % x,
|
|
'replication_ip': '10.0.0.%s' % x,
|
|
'port': base_port + x,
|
|
'replication_port': base_port + x,
|
|
'device': 'sda',
|
|
'weight': 100.0, }
|
|
x += 1
|
|
|
|
|
|
class BaseTestCompositeBuilder(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmpdir = tempfile.mkdtemp()
|
|
self.device_iter = make_device_iter()
|
|
self.output_ring = os.path.join(self.tmpdir, 'composite.ring.gz')
|
|
|
|
def pop_region_device(self, region):
|
|
dev = next(self.device_iter)
|
|
dev.update({'region': region})
|
|
return dev
|
|
|
|
def tearDown(self):
|
|
try:
|
|
shutil.rmtree(self.tmpdir, True)
|
|
except OSError:
|
|
pass
|
|
|
|
def save_builder_with_no_id(self, builder, fname):
|
|
orig_to_dict = builder.to_dict
|
|
|
|
def fake_to_dict():
|
|
res = orig_to_dict()
|
|
res.pop('id')
|
|
return res
|
|
|
|
with mock.patch.object(builder, 'to_dict', fake_to_dict):
|
|
builder.save(fname)
|
|
|
|
def save_builders(self, builders, missing_ids=None, prefix='builder'):
|
|
missing_ids = missing_ids or []
|
|
builder_files = []
|
|
for i, builder in enumerate(builders):
|
|
fname = os.path.join(self.tmpdir, '%s_%s.builder' % (prefix, i))
|
|
if i in missing_ids:
|
|
self.save_builder_with_no_id(builder, fname)
|
|
else:
|
|
builder.save(fname)
|
|
builder_files.append(fname)
|
|
return builder_files
|
|
|
|
def create_sample_ringbuilders(self, num_builders=2, rebalance=True):
|
|
"""
|
|
Create sample rings with four devices
|
|
|
|
:returns: a list of ring builder instances
|
|
"""
|
|
|
|
builders = []
|
|
for region in range(num_builders):
|
|
fname = os.path.join(self.tmpdir, 'builder_%s.builder' % region)
|
|
builder = RingBuilder(6, 3, 0)
|
|
for _ in range(5):
|
|
dev = self.pop_region_device(region)
|
|
builder.add_dev(dev)
|
|
# remove last dev to simulate a ring with some history
|
|
builder.remove_dev(dev['id'])
|
|
# add a dev that won't be assigned any parts
|
|
new_dev = self.pop_region_device(region)
|
|
new_dev['weight'] = 0
|
|
builder.add_dev(new_dev)
|
|
if rebalance:
|
|
builder.rebalance()
|
|
builder.save(fname)
|
|
self.assertTrue(os.path.exists(fname))
|
|
builders.append(builder)
|
|
|
|
return builders
|
|
|
|
def add_dev(self, builder, weight=None, region=None):
|
|
if region is None:
|
|
dev = next(builder._iter_devs())
|
|
region = dev['region']
|
|
new_dev = self.pop_region_device(region)
|
|
if weight is not None:
|
|
new_dev['weight'] = weight
|
|
builder.add_dev(new_dev)
|
|
|
|
def add_dev_and_rebalance(self, builder, weight=None):
|
|
self.add_dev(builder, weight)
|
|
builder.rebalance()
|
|
|
|
def assertDevices(self, composite_ring, builders):
|
|
"""
|
|
:param composite_ring: a Ring instance
|
|
:param builders: a list of RingBuilder instances for assertion
|
|
"""
|
|
# assert all component devices are in composite device table
|
|
builder_devs = []
|
|
for builder in builders:
|
|
builder_devs.extend([
|
|
(dev['ip'], dev['port'], dev['device'])
|
|
for dev in builder._iter_devs()])
|
|
|
|
got_devices = [
|
|
(dev['ip'], dev['port'], dev['device'])
|
|
for dev in composite_ring.devs if dev]
|
|
self.assertEqual(sorted(builder_devs), sorted(got_devices),
|
|
"composite_ring mismatched with part of the rings")
|
|
|
|
# assert composite device ids correctly index into the dev list
|
|
dev_ids = []
|
|
for i, dev in enumerate(composite_ring.devs):
|
|
if dev:
|
|
self.assertEqual(i, dev['id'])
|
|
dev_ids.append(dev['id'])
|
|
self.assertEqual(len(builder_devs), len(dev_ids))
|
|
|
|
def uniqueness(dev):
|
|
return (dev['ip'], dev['port'], dev['device'])
|
|
|
|
# assert part assignment is ordered by ring order
|
|
part_count = composite_ring.partition_count
|
|
for part in range(part_count):
|
|
primaries = [uniqueness(primary) for primary in
|
|
composite_ring.get_part_nodes(part)]
|
|
offset = 0
|
|
for builder in builders:
|
|
sub_primaries = [uniqueness(primary) for primary in
|
|
builder.get_part_devices(part)]
|
|
self.assertEqual(
|
|
primaries[offset:offset + builder.replicas],
|
|
sub_primaries,
|
|
"composite ring is not ordered by ring order, %s, %s"
|
|
% (primaries, sub_primaries))
|
|
offset += builder.replicas
|
|
|
|
def check_composite_ring(self, ring_file, builders):
|
|
got_ring = Ring(ring_file)
|
|
self.assertEqual(got_ring.partition_count, builders[0].parts)
|
|
self.assertEqual(got_ring.replica_count,
|
|
sum(b.replicas for b in builders))
|
|
self.assertEqual(got_ring._part_shift, builders[0].part_shift)
|
|
self.assertDevices(got_ring, builders)
|
|
|
|
def check_composite_meta(self, cb_file, builder_files, version=1):
|
|
with open(cb_file) as fd:
|
|
actual = json.load(fd)
|
|
builders = [RingBuilder.load(fname) for fname in builder_files]
|
|
expected_metadata = {
|
|
'saved_path': os.path.abspath(cb_file),
|
|
'serialization_version': 1,
|
|
'version': version,
|
|
'components': [
|
|
{'id': builder.id,
|
|
'version': builder.version,
|
|
'replicas': builder.replicas,
|
|
}
|
|
for builder in builders
|
|
],
|
|
'component_builder_files':
|
|
dict((builder.id, os.path.abspath(builder_files[i]))
|
|
for i, builder in enumerate(builders))
|
|
}
|
|
self.assertEqual(expected_metadata, actual)
|
|
|
|
def _make_composite_builder(self, builders):
|
|
# helper to compose a ring, save it and sanity check it
|
|
builder_files = self.save_builders(builders)
|
|
cb = CompositeRingBuilder(builder_files)
|
|
cb.compose().save(self.output_ring)
|
|
self.check_composite_ring(self.output_ring, builders)
|
|
return cb, builder_files
|
|
|
|
|
|
class TestCompositeBuilder(BaseTestCompositeBuilder):
|
|
def test_compose_rings(self):
|
|
def do_test(builder_count):
|
|
builders = self.create_sample_ringbuilders(builder_count)
|
|
rd = compose_rings(builders)
|
|
rd.save(self.output_ring)
|
|
self.check_composite_ring(self.output_ring, builders)
|
|
|
|
do_test(2)
|
|
do_test(3)
|
|
do_test(4)
|
|
|
|
def test_composite_same_region_in_the_different_rings_error(self):
|
|
builder_1 = self.create_sample_ringbuilders(1)
|
|
builder_2 = self.create_sample_ringbuilders(1)
|
|
builders = builder_1 + builder_2
|
|
with self.assertRaises(ValueError) as cm:
|
|
compose_rings(builders)
|
|
self.assertIn('Same region found in different rings',
|
|
cm.exception.message)
|
|
|
|
def test_composite_only_one_ring_in_the_args_error(self):
|
|
builders = self.create_sample_ringbuilders(1)
|
|
with self.assertRaises(ValueError) as cm:
|
|
compose_rings(builders)
|
|
self.assertIn(
|
|
'Two or more component builders are required.',
|
|
cm.exception.message)
|
|
|
|
def test_composite_same_device_in_the_different_rings_error(self):
|
|
builders = self.create_sample_ringbuilders(2)
|
|
same_device = copy.deepcopy(builders[0].devs[0])
|
|
|
|
# create one more ring which duplicates a device in the first ring
|
|
builder = RingBuilder(6, 3, 1)
|
|
_, fname = tempfile.mkstemp(dir=self.tmpdir)
|
|
# add info to feed to add_dev
|
|
same_device.update({'region': 2, 'weight': 100})
|
|
builder.add_dev(same_device)
|
|
|
|
# add rest of the devices, which are unique
|
|
for _ in range(3):
|
|
dev = self.pop_region_device(2)
|
|
builder.add_dev(dev)
|
|
builder.rebalance()
|
|
builder.save(fname)
|
|
# sanity
|
|
self.assertTrue(os.path.exists(fname))
|
|
|
|
builders.append(builder)
|
|
|
|
with self.assertRaises(ValueError) as cm:
|
|
compose_rings(builders)
|
|
self.assertIn(
|
|
'Duplicate ip/port/device combination %(ip)s/%(port)s/%(device)s '
|
|
'found in builders at indexes 0 and 2' %
|
|
same_device, cm.exception.message)
|
|
|
|
def test_different_part_power_error(self):
|
|
# create a ring builder
|
|
# (default, part power is 6 with create_sample_ringbuilders)
|
|
builders = self.create_sample_ringbuilders(1)
|
|
|
|
# prepare another ring which has different part power
|
|
incorrect_builder = RingBuilder(4, 3, 1)
|
|
_, fname = tempfile.mkstemp(dir=self.tmpdir)
|
|
for _ in range(4):
|
|
dev = self.pop_region_device(1)
|
|
incorrect_builder.add_dev(dev)
|
|
incorrect_builder.rebalance()
|
|
incorrect_builder.save(fname)
|
|
# sanity
|
|
self.assertTrue(os.path.exists(fname))
|
|
|
|
# sanity
|
|
correct_builder = builders[0]
|
|
self.assertNotEqual(correct_builder.part_shift,
|
|
incorrect_builder.part_shift)
|
|
self.assertNotEqual(correct_builder.part_power,
|
|
incorrect_builder.part_power)
|
|
|
|
builders.append(incorrect_builder)
|
|
with self.assertRaises(ValueError) as cm:
|
|
compose_rings(builders)
|
|
self.assertIn("All builders must have same value for 'part_power'",
|
|
cm.exception.message)
|
|
|
|
def test_compose_rings_float_replica_count_builder_error(self):
|
|
builders = self.create_sample_ringbuilders(1)
|
|
|
|
# prepare another ring which has float replica count
|
|
incorrect_builder = RingBuilder(6, 1.5, 1)
|
|
_, fname = tempfile.mkstemp(dir=self.tmpdir)
|
|
for _ in range(4):
|
|
dev = self.pop_region_device(1)
|
|
incorrect_builder.add_dev(dev)
|
|
incorrect_builder.rebalance()
|
|
incorrect_builder.save(fname)
|
|
# sanity
|
|
self.assertTrue(os.path.exists(fname))
|
|
self.assertEqual(1.5, incorrect_builder.replicas)
|
|
# the first replica has 2 ** 6 partitions
|
|
self.assertEqual(
|
|
2 ** 6, len(incorrect_builder._replica2part2dev[0]))
|
|
# but the second replica has the half of the first partitions
|
|
self.assertEqual(
|
|
2 ** 5, len(incorrect_builder._replica2part2dev[1]))
|
|
builders.append(incorrect_builder)
|
|
|
|
with self.assertRaises(ValueError) as cm:
|
|
compose_rings(builders)
|
|
self.assertIn("Problem with builders", cm.exception.message)
|
|
self.assertIn("Non integer replica count", cm.exception.message)
|
|
|
|
def test_compose_rings_rebalance_needed(self):
|
|
builders = self.create_sample_ringbuilders(2)
|
|
|
|
# add a new device to builder 1 but no rebalance
|
|
dev = self.pop_region_device(1)
|
|
builders[1].add_dev(dev)
|
|
self.assertTrue(builders[1].devs_changed) # sanity check
|
|
with self.assertRaises(ValueError) as cm:
|
|
compose_rings(builders)
|
|
self.assertIn("Problem with builders", cm.exception.message)
|
|
self.assertIn("Builder needs rebalance", cm.exception.message)
|
|
# after rebalance, that works (sanity)
|
|
builders[1].rebalance()
|
|
compose_rings(builders)
|
|
|
|
def test_different_replica_count_works(self):
|
|
# create a ring builder
|
|
# (default, part power is 6 with create_sample_ringbuilders)
|
|
builders = self.create_sample_ringbuilders(1)
|
|
|
|
# prepare another ring which has different replica count
|
|
builder = RingBuilder(6, 1, 1)
|
|
_, fname = tempfile.mkstemp(dir=self.tmpdir)
|
|
for _ in range(4):
|
|
dev = self.pop_region_device(1)
|
|
builder.add_dev(dev)
|
|
builder.rebalance()
|
|
builder.save(fname)
|
|
# sanity
|
|
self.assertTrue(os.path.exists(fname))
|
|
builders.append(builder)
|
|
|
|
rd = compose_rings(builders)
|
|
rd.save(self.output_ring)
|
|
got_ring = Ring(self.output_ring)
|
|
self.assertEqual(got_ring.partition_count, 2 ** 6)
|
|
self.assertEqual(got_ring.replica_count, 4) # 3 + 1
|
|
self.assertEqual(got_ring._part_shift, 26)
|
|
self.assertDevices(got_ring, builders)
|
|
|
|
def test_ring_swap(self):
|
|
# sanity
|
|
builders = sorted(self.create_sample_ringbuilders(2))
|
|
rd = compose_rings(builders)
|
|
rd.save(self.output_ring)
|
|
got_ring = Ring(self.output_ring)
|
|
self.assertEqual(got_ring.partition_count, 2 ** 6)
|
|
self.assertEqual(got_ring.replica_count, 6)
|
|
self.assertEqual(got_ring._part_shift, 26)
|
|
self.assertDevices(got_ring, builders)
|
|
|
|
# even if swapped, it works
|
|
reverse_builders = sorted(builders, reverse=True)
|
|
self.assertNotEqual(reverse_builders, builders)
|
|
rd = compose_rings(reverse_builders)
|
|
rd.save(self.output_ring)
|
|
got_ring = Ring(self.output_ring)
|
|
self.assertEqual(got_ring.partition_count, 2 ** 6)
|
|
self.assertEqual(got_ring.replica_count, 6)
|
|
self.assertEqual(got_ring._part_shift, 26)
|
|
self.assertDevices(got_ring, reverse_builders)
|
|
|
|
# but if the composite rings are different order, the composite ring
|
|
# *will* be different. Note that the CompositeRingBuilder class will
|
|
# check builder order against the existing ring and fail if the order
|
|
# is different (actually checking the metadata). See also
|
|
# test_compose_different_builder_order
|
|
with self.assertRaises(AssertionError) as cm:
|
|
self.assertDevices(got_ring, builders)
|
|
|
|
self.assertIn("composite ring is not ordered by ring order",
|
|
cm.exception.message)
|
|
|
|
|
|
class TestCompositeRingBuilder(BaseTestCompositeBuilder):
|
|
def test_compose_with_builder_files(self):
|
|
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json')
|
|
builders = self.create_sample_ringbuilders(2)
|
|
cb, _ = self._make_composite_builder(builders)
|
|
cb.save(cb_file)
|
|
|
|
for i, b in enumerate(builders):
|
|
self.add_dev_and_rebalance(b)
|
|
self.save_builders(builders)
|
|
cb = CompositeRingBuilder.load(cb_file)
|
|
cb.compose().save(self.output_ring)
|
|
self.check_composite_ring(self.output_ring, builders)
|
|
|
|
def test_compose_ok(self):
|
|
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json')
|
|
builders = self.create_sample_ringbuilders(2)
|
|
# make first version of composite ring
|
|
cb, builder_files = self._make_composite_builder(builders)
|
|
# check composite builder persists ok
|
|
cb.save(cb_file)
|
|
self.assertTrue(os.path.exists(cb_file))
|
|
self.check_composite_meta(cb_file, builder_files)
|
|
# and reloads ok
|
|
cb = CompositeRingBuilder.load(cb_file)
|
|
self.assertEqual(1, cb.version)
|
|
# compose detects if no component builder changes, if we ask it to...
|
|
with self.assertRaises(ValueError) as cm:
|
|
cb.compose(require_modified=True)
|
|
self.assertIn('None of the component builders has been modified',
|
|
cm.exception.message)
|
|
self.assertEqual(1, cb.version)
|
|
# ...but by default will compose again despite no changes to components
|
|
cb.compose(force=True).save(self.output_ring)
|
|
self.check_composite_ring(self.output_ring, builders)
|
|
self.assertEqual(2, cb.version)
|
|
# check composite builder persists ok again
|
|
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json2')
|
|
cb.save(cb_file)
|
|
self.assertTrue(os.path.exists(cb_file))
|
|
self.check_composite_meta(cb_file, builder_files, version=2)
|
|
|
|
def test_compose_modified_component_builders(self):
|
|
# check it's ok to compose again with same but modified builders
|
|
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json')
|
|
builders = self.create_sample_ringbuilders(2)
|
|
cb, builder_files = self._make_composite_builder(builders)
|
|
ring = Ring(self.output_ring)
|
|
orig_devs = [dev for dev in ring.devs if dev]
|
|
self.assertEqual(10, len(orig_devs)) # sanity check
|
|
self.add_dev_and_rebalance(builders[1])
|
|
builder_files = self.save_builders(builders)
|
|
cb.compose().save(self.output_ring)
|
|
self.check_composite_ring(self.output_ring, builders)
|
|
ring = Ring(self.output_ring)
|
|
modified_devs = [dev for dev in ring.devs if dev]
|
|
self.assertEqual(len(orig_devs) + 1, len(modified_devs))
|
|
# check composite builder persists ok
|
|
cb.save(cb_file)
|
|
self.assertTrue(os.path.exists(cb_file))
|
|
self.check_composite_meta(cb_file, builder_files, version=2)
|
|
# and reloads ok
|
|
cb = CompositeRingBuilder.load(cb_file)
|
|
# and composes ok after reload
|
|
cb.compose(force=True).save(self.output_ring)
|
|
self.check_composite_ring(self.output_ring, builders)
|
|
# check composite builder persists ok again
|
|
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json2')
|
|
cb.save(cb_file)
|
|
self.assertTrue(os.path.exists(cb_file))
|
|
self.check_composite_meta(cb_file, builder_files, version=3)
|
|
|
|
def test_compose_override_component_builders(self):
|
|
# check passing different builder files to the compose() method
|
|
# overrides loaded builder files
|
|
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json')
|
|
builders = self.create_sample_ringbuilders(2)
|
|
cb, builder_files = self._make_composite_builder(builders)
|
|
# modify builders and save in different files
|
|
self.add_dev_and_rebalance(builders[1])
|
|
with self.assertRaises(ValueError):
|
|
# sanity check - originals are unchanged
|
|
cb.compose(builder_files, require_modified=True)
|
|
other_files = self.save_builders(builders, prefix='other')
|
|
cb.compose(other_files, require_modified=True).save(self.output_ring)
|
|
self.check_composite_ring(self.output_ring, builders)
|
|
# check composite builder persists ok
|
|
cb.save(cb_file)
|
|
self.assertTrue(os.path.exists(cb_file))
|
|
self.check_composite_meta(cb_file, other_files, version=2)
|
|
# and reloads ok
|
|
cb = CompositeRingBuilder.load(cb_file)
|
|
# and composes ok after reload
|
|
cb.compose(force=True).save(self.output_ring)
|
|
self.check_composite_ring(self.output_ring, builders)
|
|
# check composite builder persists ok again
|
|
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json2')
|
|
cb.save(cb_file)
|
|
self.assertTrue(os.path.exists(cb_file))
|
|
self.check_composite_meta(cb_file, other_files, version=3)
|
|
|
|
def test_abs_paths_persisted(self):
|
|
cwd = os.getcwd()
|
|
try:
|
|
os.chdir(self.tmpdir)
|
|
builders = self.create_sample_ringbuilders(2)
|
|
builder_files = self.save_builders(builders)
|
|
rel_builder_files = [os.path.basename(bf) for bf in builder_files]
|
|
cb = CompositeRingBuilder(rel_builder_files)
|
|
cb.compose().save(self.output_ring)
|
|
self.check_composite_ring(self.output_ring, builders)
|
|
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json')
|
|
rel_cb_file = os.path.basename(cb_file)
|
|
cb.save(rel_cb_file)
|
|
self.check_composite_meta(rel_cb_file, rel_builder_files)
|
|
finally:
|
|
os.chdir(cwd)
|
|
|
|
def test_load_errors(self):
|
|
bad_file = os.path.join(self.tmpdir, 'bad_file.json')
|
|
with self.assertRaises(IOError):
|
|
CompositeRingBuilder.load(bad_file)
|
|
|
|
def check_bad_content(content):
|
|
with open(bad_file, 'wb') as fp:
|
|
fp.write(content)
|
|
try:
|
|
with self.assertRaises(ValueError) as cm:
|
|
CompositeRingBuilder.load(bad_file)
|
|
self.assertIn(
|
|
"File does not contain valid composite ring data",
|
|
cm.exception.message)
|
|
except AssertionError as err:
|
|
raise AssertionError('With content %r: %s' % (content, err))
|
|
|
|
for content in ('', 'not json', json.dumps({}), json.dumps([])):
|
|
check_bad_content(content)
|
|
|
|
good_content = {
|
|
'components': [
|
|
{'version': 1, 'id': 'uuid_x', 'replicas': 12},
|
|
{'version': 2, 'id': 'uuid_y', 'replicas': 12}
|
|
],
|
|
'builder_files': {'uuid_x': '/path/to/file_x',
|
|
'uuid_y': '/path/to/file_y'},
|
|
'version': 99}
|
|
for missing in good_content:
|
|
bad_content = dict(good_content)
|
|
bad_content.pop(missing)
|
|
check_bad_content(json.dumps(bad_content))
|
|
|
|
def test_save_errors(self):
|
|
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json')
|
|
|
|
def do_test(cb):
|
|
with self.assertRaises(ValueError) as cm:
|
|
cb.save(cb_file)
|
|
self.assertIn("No composed ring to save", cm.exception.message)
|
|
|
|
do_test(CompositeRingBuilder())
|
|
do_test(CompositeRingBuilder([]))
|
|
do_test(CompositeRingBuilder(['file1', 'file2']))
|
|
|
|
def test_rebalance(self):
|
|
@contextmanager
|
|
def mock_rebalance():
|
|
# captures component builder rebalance call results, yields a dict
|
|
# that maps builder -> results
|
|
calls = defaultdict(list)
|
|
orig_func = RingBuilder.rebalance
|
|
|
|
def func(builder, **kwargs):
|
|
result = orig_func(builder, **kwargs)
|
|
calls[builder].append(result)
|
|
return result
|
|
|
|
with mock.patch('swift.common.ring.RingBuilder.rebalance', func):
|
|
yield calls
|
|
|
|
def check_results():
|
|
self.assertEqual(2, len(rebalance_calls)) # 2 builders called
|
|
for calls in rebalance_calls.values():
|
|
self.assertFalse(calls[1:]) # 1 call to each builder
|
|
|
|
self.assertEqual(sorted(expected_ids),
|
|
sorted([b.id for b in rebalance_calls]))
|
|
self.assertEqual(sorted(expected_versions),
|
|
sorted([b.version for b in rebalance_calls]))
|
|
for b in rebalance_calls:
|
|
self.assertEqual(set(rebalance_calls.keys()),
|
|
set(b.parent_builder._builders))
|
|
|
|
# check the rebalanced builders were saved
|
|
written_builders = [RingBuilder.load(f) for f in builder_files]
|
|
self.assertEqual(expected_ids,
|
|
[b.id for b in written_builders])
|
|
self.assertEqual(expected_versions,
|
|
[b.version for b in written_builders])
|
|
|
|
# check returned results, should be in component order
|
|
self.assertEqual(2, len(results))
|
|
self.assertEqual(builder_files,
|
|
[r['builder_file'] for r in results])
|
|
self.assertEqual(expected_versions,
|
|
[r['builder'].version for r in results])
|
|
self.assertEqual(expected_ids, [r['builder'].id for r in results])
|
|
self.assertEqual(
|
|
[rebalance_calls[r['builder']][0] for r in results],
|
|
[r['result'] for r in results])
|
|
|
|
# N.B. the sample builders have zero min_part_hours
|
|
builders = self.create_sample_ringbuilders(2)
|
|
expected_versions = [b.version + 1 for b in builders]
|
|
expected_ids = [b.id for b in builders]
|
|
|
|
# test rebalance loads component builders
|
|
builder_files = self.save_builders(builders)
|
|
cb = CompositeRingBuilder(builder_files)
|
|
with mock_rebalance() as rebalance_calls:
|
|
results = cb.rebalance()
|
|
check_results()
|
|
|
|
# test loading builder files via load_components
|
|
# revert builder files to original builder state
|
|
builder_files = self.save_builders(builders)
|
|
cb = CompositeRingBuilder()
|
|
cb.load_components(builder_files)
|
|
with mock_rebalance() as rebalance_calls:
|
|
results = cb.rebalance()
|
|
check_results()
|
|
|
|
def test_rebalance_errors(self):
|
|
cb = CompositeRingBuilder()
|
|
with self.assertRaises(ValueError) as cm:
|
|
cb.rebalance()
|
|
self.assertIn('Two or more component builders are required',
|
|
cm.exception.message)
|
|
|
|
builders = self.create_sample_ringbuilders(2)
|
|
cb, builder_files = self._make_composite_builder(builders)
|
|
with mock.patch('swift.common.ring.RingBuilder.rebalance',
|
|
side_effect=RingBuilderError('test')):
|
|
with mock.patch('swift.common.ring.composite_builder.shuffle',
|
|
lambda x: x):
|
|
with self.assertRaises(RingBuilderError) as cm:
|
|
cb.rebalance()
|
|
self.assertIn('An error occurred while rebalancing component %s' %
|
|
builder_files[0], str(cm.exception))
|
|
self.assertIsNone(cb._builders)
|
|
|
|
with mock.patch('swift.common.ring.RingBuilder.validate',
|
|
side_effect=RingBuilderError('test')):
|
|
with mock.patch('swift.common.ring.composite_builder.shuffle',
|
|
lambda x: x):
|
|
with self.assertRaises(RingBuilderError) as cm:
|
|
cb.rebalance()
|
|
self.assertIn('An error occurred while rebalancing component %s' %
|
|
builder_files[0], str(cm.exception))
|
|
self.assertIsNone(cb._builders)
|
|
|
|
def test_rebalance_with_unrebalanced_builders(self):
|
|
# create 2 non-rebalanced rings
|
|
builders = self.create_sample_ringbuilders(rebalance=False)
|
|
# save builders
|
|
builder_files = self.save_builders(builders)
|
|
cb = CompositeRingBuilder(builder_files)
|
|
# sanity, it is impossible to compose un-rebalanced component rings
|
|
with self.assertRaises(ValueError) as cm:
|
|
cb.compose()
|
|
self.assertIn("Builder needs rebalance", cm.exception.message)
|
|
# but ok to compose after rebalance
|
|
cb.rebalance()
|
|
rd = cb.compose()
|
|
rd.save(self.output_ring)
|
|
rebalanced_builders = [RingBuilder.load(f) for f in builder_files]
|
|
self.check_composite_ring(self.output_ring, rebalanced_builders)
|
|
|
|
|
|
class TestLoadComponents(BaseTestCompositeBuilder):
|
|
# Tests for the loading of component builders.
|
|
def _call_method_under_test(self, cb, *args, **kwargs):
|
|
# Component builder loading is triggered by the load_components method
|
|
# and the compose method. This method provides a hook for subclasses to
|
|
# configure a different method to repeat the component loading tests.
|
|
cb.load_components(*args, **kwargs)
|
|
|
|
def test_load_components(self):
|
|
builders = self.create_sample_ringbuilders(2)
|
|
builder_files = self.save_builders(builders)
|
|
cb = CompositeRingBuilder(builder_files)
|
|
# check lazy loading
|
|
self.assertEqual(builder_files, cb._builder_files)
|
|
self.assertFalse(cb._builders) # none loaded yet
|
|
|
|
# check loading configured files
|
|
self._call_method_under_test(cb)
|
|
self.assertEqual(builder_files, cb._builder_files)
|
|
for i, builder in enumerate(cb._builders):
|
|
self.assertEqual(builders[i].id, builder.id)
|
|
self.assertEqual(builders[i].devs, builder.devs)
|
|
|
|
# modify builders and save in different files
|
|
self.add_dev_and_rebalance(builders[0])
|
|
other_files = self.save_builders(builders, prefix='other')
|
|
# reload from other files
|
|
self._call_method_under_test(cb, other_files)
|
|
self.assertEqual(other_files, cb._builder_files)
|
|
for i, builder in enumerate(cb._builders):
|
|
self.assertEqual(builders[i].id, builder.id)
|
|
self.assertEqual(builders[i].devs, builder.devs)
|
|
|
|
# modify builders again and save in same files
|
|
self.add_dev_and_rebalance(builders[1])
|
|
self.save_builders(builders, prefix='other')
|
|
# reload from same files
|
|
self._call_method_under_test(cb)
|
|
self.assertEqual(other_files, cb._builder_files)
|
|
for i, builder in enumerate(cb._builders):
|
|
self.assertEqual(builders[i].id, builder.id)
|
|
self.assertEqual(builders[i].devs, builder.devs)
|
|
|
|
def test_load_components_insufficient_builders(self):
|
|
def do_test(builder_files, force):
|
|
cb = CompositeRingBuilder(builder_files)
|
|
with self.assertRaises(ValueError) as cm:
|
|
self._call_method_under_test(cb, builder_files,
|
|
force=force)
|
|
self.assertIn('Two or more component builders are required',
|
|
cm.exception.message)
|
|
|
|
cb = CompositeRingBuilder()
|
|
with self.assertRaises(ValueError) as cm:
|
|
self._call_method_under_test(cb, builder_files,
|
|
force=force)
|
|
self.assertIn('Two or more component builders are required',
|
|
cm.exception.message)
|
|
|
|
builders = self.create_sample_ringbuilders(3)
|
|
builder_files = self.save_builders(builders)
|
|
do_test([], force=False)
|
|
do_test([], force=True) # this error is never ignored
|
|
do_test(builder_files[:1], force=False)
|
|
do_test(builder_files[:1], force=True) # this error is never ignored
|
|
|
|
def test_load_components_missing_builder_id(self):
|
|
def check_missing_id(cb, builders):
|
|
# not ok to load builder_files that have no id assigned
|
|
orig_version = cb.version
|
|
no_id = random.randint(0, len(builders) - 1)
|
|
# rewrite the builder files so that one has missing id
|
|
builder_files = self.save_builders(builders, missing_ids=[no_id])
|
|
|
|
def do_check(force):
|
|
with self.assertRaises(ValueError) as cm:
|
|
self._call_method_under_test(cb, builder_files,
|
|
force=force)
|
|
error_lines = cm.exception.message.split('\n')
|
|
self.assertIn("Problem with builder at index %s" % no_id,
|
|
error_lines[0])
|
|
self.assertIn("id attribute has not been initialised",
|
|
error_lines[0])
|
|
self.assertFalse(error_lines[1:])
|
|
self.assertEqual(orig_version, cb.version)
|
|
|
|
do_check(False)
|
|
do_check(True) # we never ignore this error
|
|
|
|
# check with compose not previously called, cb has no existing metadata
|
|
builders = self.create_sample_ringbuilders(3)
|
|
cb = CompositeRingBuilder()
|
|
check_missing_id(cb, builders)
|
|
# now save good copies of builders and compose so this cb has
|
|
# existing component metadata
|
|
builder_files = self.save_builders(builders)
|
|
cb = CompositeRingBuilder(builder_files)
|
|
cb.compose() # cb now has component metadata
|
|
check_missing_id(cb, builders)
|
|
|
|
def test_load_components_duplicate_builder_ids(self):
|
|
builders = self.create_sample_ringbuilders(3)
|
|
builders[2]._id = builders[0]._id
|
|
cb = CompositeRingBuilder(self.save_builders(builders))
|
|
|
|
def do_check(force):
|
|
with self.assertRaises(ValueError) as cm:
|
|
self._call_method_under_test(cb, force=force)
|
|
error_lines = cm.exception.message.split('\n')
|
|
self.assertIn("Builder id %r used at indexes 0, 2" %
|
|
builders[0].id, error_lines[0])
|
|
self.assertFalse(error_lines[1:])
|
|
self.assertEqual(0, cb.version)
|
|
|
|
do_check(False)
|
|
do_check(True)
|
|
|
|
def test_load_components_unchanged_builders(self):
|
|
def do_test(cb, builder_files, **kwargs):
|
|
orig_version = cb.version
|
|
with self.assertRaises(ValueError) as cm:
|
|
self._call_method_under_test(cb, builder_files, **kwargs)
|
|
error_lines = cm.exception.message.split('\n')
|
|
self.assertIn("None of the component builders has been modified",
|
|
error_lines[0])
|
|
self.assertFalse(error_lines[1:])
|
|
self.assertEqual(orig_version, cb.version)
|
|
|
|
builders = self.create_sample_ringbuilders(2)
|
|
cb, builder_files = self._make_composite_builder(builders)
|
|
# ok to load same *unchanged* builders
|
|
self._call_method_under_test(cb, builder_files)
|
|
# unless require_modified is set
|
|
do_test(cb, builder_files, require_modified=True)
|
|
# even if we rewrite the files
|
|
builder_files = self.save_builders(builders)
|
|
do_test(cb, builder_files, require_modified=True)
|
|
# even if we rename the files
|
|
builder_files = self.save_builders(builders, prefix='other')
|
|
do_test(cb, builder_files, require_modified=True)
|
|
# force trumps require_modified
|
|
self._call_method_under_test(cb, builder_files, force=True,
|
|
require_modified=True)
|
|
|
|
def test_load_components_older_builder(self):
|
|
# make first version of composite ring
|
|
builders = self.create_sample_ringbuilders(2)
|
|
cb, builder_files = self._make_composite_builder(builders)
|
|
old_builders = [copy.deepcopy(b) for b in builders]
|
|
# update components and reload
|
|
for i, b in enumerate(builders):
|
|
self.add_dev_and_rebalance(b)
|
|
self.assertLess(old_builders[i].version, b.version)
|
|
self.save_builders(builders)
|
|
self._call_method_under_test(cb)
|
|
orig_version = cb.version
|
|
cb.compose() # compose with newer builder versions
|
|
self.assertEqual(orig_version + 1, cb.version) # sanity check
|
|
# not ok to use old versions of same builders
|
|
self.save_builders([old_builders[0], builders[1]])
|
|
with self.assertRaises(ValueError) as cm:
|
|
self._call_method_under_test(cb)
|
|
error_lines = cm.exception.message.split('\n')
|
|
self.assertIn("Invalid builder change at index 0", error_lines[0])
|
|
self.assertIn("Older builder version", error_lines[0])
|
|
self.assertFalse(error_lines[1:])
|
|
self.assertEqual(orig_version + 1, cb.version)
|
|
# not even if one component ring has changed
|
|
self.add_dev_and_rebalance(builders[1])
|
|
self.save_builders([old_builders[0], builders[1]])
|
|
with self.assertRaises(ValueError) as cm:
|
|
self._call_method_under_test(cb)
|
|
error_lines = cm.exception.message.split('\n')
|
|
self.assertIn("Invalid builder change at index 0", error_lines[0])
|
|
self.assertIn("Older builder version", error_lines[0])
|
|
self.assertFalse(error_lines[1:])
|
|
self.assertEqual(orig_version + 1, cb.version)
|
|
self.assertIsNone(cb._builders)
|
|
# unless we ignore errors
|
|
self._call_method_under_test(cb, force=True)
|
|
self.assertEqual(old_builders[0].version, cb._builders[0].version)
|
|
|
|
def test_load_components_different_number_builders(self):
|
|
# not ok to use a different number of component rings
|
|
builders = self.create_sample_ringbuilders(4)
|
|
|
|
def do_test(bad_builders):
|
|
cb, builder_files = self._make_composite_builder(builders[:3])
|
|
# expect an error
|
|
with self.assertRaises(ValueError) as cm:
|
|
self._call_method_under_test(
|
|
cb, self.save_builders(bad_builders))
|
|
error_lines = cm.exception.message.split('\n')
|
|
self.assertFalse(error_lines[1:])
|
|
self.assertEqual(1, cb.version)
|
|
# unless we ignore errors
|
|
self._call_method_under_test(cb, self.save_builders(bad_builders),
|
|
force=True)
|
|
self.assertEqual(len(bad_builders), len(cb._builders))
|
|
return error_lines
|
|
|
|
error_lines = do_test(builders[:2]) # too few
|
|
self.assertIn("Missing builder at index 2", error_lines[0])
|
|
error_lines = do_test(builders) # too many
|
|
self.assertIn("Unexpected extra builder at index 3", error_lines[0])
|
|
|
|
def test_load_components_different_builders(self):
|
|
# not ok to change component rings
|
|
builders = self.create_sample_ringbuilders(3)
|
|
cb, builder_files = self._make_composite_builder(builders[:2])
|
|
# ensure builder[0] is newer version so that's not the problem
|
|
self.add_dev_and_rebalance(builders[0])
|
|
different_files = self.save_builders([builders[0], builders[2]])
|
|
with self.assertRaises(ValueError) as cm:
|
|
self._call_method_under_test(cb, different_files)
|
|
error_lines = cm.exception.message.split('\n')
|
|
self.assertIn("Invalid builder change at index 1", error_lines[0])
|
|
self.assertIn("Attribute mismatch for id", error_lines[0])
|
|
self.assertFalse(error_lines[1:])
|
|
self.assertEqual(1, cb.version)
|
|
# ok if we force
|
|
self._call_method_under_test(cb, different_files, force=True)
|
|
self.assertEqual(different_files, cb._builder_files)
|
|
|
|
def test_load_component_different_builder_order(self):
|
|
# not ok to change order of component rings
|
|
builders = self.create_sample_ringbuilders(4)
|
|
cb, builder_files = self._make_composite_builder(builders)
|
|
builder_files.reverse()
|
|
with self.assertRaises(ValueError) as cm:
|
|
self._call_method_under_test(cb, builder_files)
|
|
error_lines = cm.exception.message.split('\n')
|
|
for i, line in enumerate(error_lines):
|
|
self.assertIn("Invalid builder change at index %s" % i, line)
|
|
self.assertIn("Attribute mismatch for id", line)
|
|
self.assertEqual(1, cb.version)
|
|
# ok if we force
|
|
self._call_method_under_test(cb, builder_files, force=True)
|
|
self.assertEqual(builder_files, cb._builder_files)
|
|
|
|
def test_load_components_replica_count_changed(self):
|
|
# not ok to change the number of replicas in a ring
|
|
builders = self.create_sample_ringbuilders(3)
|
|
cb, builder_files = self._make_composite_builder(builders)
|
|
builders[0].set_replicas(4)
|
|
self.save_builders(builders)
|
|
with self.assertRaises(ValueError) as cm:
|
|
self._call_method_under_test(cb)
|
|
error_lines = cm.exception.message.split('\n')
|
|
for i, line in enumerate(error_lines):
|
|
self.assertIn("Invalid builder change at index 0", line)
|
|
self.assertIn("Attribute mismatch for replicas", line)
|
|
self.assertEqual(1, cb.version)
|
|
# ok if we force
|
|
self._call_method_under_test(cb, force=True)
|
|
|
|
|
|
class TestComposeLoadComponents(TestLoadComponents):
|
|
def _call_method_under_test(self, cb, *args, **kwargs):
|
|
cb.compose(*args, **kwargs)
|
|
|
|
def test_load_components_replica_count_changed(self):
|
|
# For compose method this test differs from superclass when the force
|
|
# flag is used, because although the force flag causes load_components
|
|
# to skip checks, the actual ring composition fails.
|
|
# not ok to change the number of replicas in a ring
|
|
builders = self.create_sample_ringbuilders(3)
|
|
cb, builder_files = self._make_composite_builder(builders)
|
|
builders[0].set_replicas(4)
|
|
self.save_builders(builders)
|
|
with self.assertRaises(ValueError) as cm:
|
|
self._call_method_under_test(cb)
|
|
error_lines = cm.exception.message.split('\n')
|
|
for i, line in enumerate(error_lines):
|
|
self.assertIn("Invalid builder change at index 0", line)
|
|
self.assertIn("Attribute mismatch for replicas", line)
|
|
self.assertEqual(1, cb.version)
|
|
# if we force, then load_components succeeds but the compose pre
|
|
# validate will fail because the builder needs rebalancing
|
|
with self.assertRaises(ValueError) as cm:
|
|
self._call_method_under_test(cb, force=True)
|
|
error_lines = cm.exception.message.split('\n')
|
|
self.assertIn("Problem with builders", error_lines[0])
|
|
self.assertIn("Builder needs rebalance", error_lines[1])
|
|
self.assertFalse(error_lines[2:])
|
|
self.assertEqual(1, cb.version)
|
|
|
|
|
|
class TestCooperativeRingBuilder(BaseTestCompositeBuilder):
|
|
def _make_coop_builder(self, region, composite_builder, rebalance=False,
|
|
min_part_hours=1):
|
|
rb = CooperativeRingBuilder(8, 3, min_part_hours, composite_builder)
|
|
if composite_builder._builders is None:
|
|
composite_builder._builders = [rb]
|
|
for i in range(3):
|
|
self.add_dev(rb, region=region)
|
|
if rebalance:
|
|
rb.rebalance()
|
|
self.assertEqual(self._partition_counts(rb),
|
|
{0: 256, 1: 256, 2: 256}) # sanity check
|
|
return rb
|
|
|
|
def _partition_counts(self, builder):
|
|
"""
|
|
Returns a dictionary mapping device id's to (number of
|
|
partitions assigned to that device).
|
|
"""
|
|
return Counter(builder.devs[dev_id]['id']
|
|
for part2dev_id in builder._replica2part2dev
|
|
for dev_id in part2dev_id)
|
|
|
|
def get_moved_parts(self, after, before):
|
|
def uniqueness(dev):
|
|
return dev['ip'], dev['port'], dev['device']
|
|
moved_parts = set()
|
|
for p in range(before.parts):
|
|
if ({uniqueness(dev) for dev in before._devs_for_part(p)} !=
|
|
{uniqueness(dev) for dev in after._devs_for_part(p)}):
|
|
moved_parts.add(p)
|
|
return moved_parts
|
|
|
|
def num_parts_can_move(self, builder):
|
|
# note that can_part_move() gives consideration to the
|
|
# _part_moved_bitmap which is only reset when a rebalance starts
|
|
return len(
|
|
[p for p in range(builder.parts)
|
|
if super(CooperativeRingBuilder, builder)._can_part_move(p)])
|
|
|
|
@mock.patch('swift.common.ring.builder.time')
|
|
def _check_rebalance_respects_cobuilder_part_moves(
|
|
self, min_part_hours, mock_time):
|
|
mock_time.return_value = now = int(time.time())
|
|
builder_files = []
|
|
cb = CompositeRingBuilder()
|
|
for i in (1, 2, 3):
|
|
b = self._make_coop_builder(i, cb, min_part_hours=min_part_hours)
|
|
fname = os.path.join(self.tmpdir, 'builder_%s.builder' % i)
|
|
b.save(fname)
|
|
builder_files.append(fname)
|
|
builder_files, builders = cb.load_components(builder_files)
|
|
|
|
# all cobuilders can perform initial rebalance
|
|
cb.rebalance()
|
|
exp = {0: 256, 1: 256, 2: 256}
|
|
self.assertEqual(exp, self._partition_counts(builders[0]))
|
|
self.assertEqual(exp, self._partition_counts(builders[1]))
|
|
self.assertEqual(exp, self._partition_counts(builders[2]))
|
|
exp = min_part_hours * 3600
|
|
self.assertEqual(exp, builders[0].min_part_seconds_left)
|
|
self.assertEqual(exp, builders[1].min_part_seconds_left)
|
|
self.assertEqual(exp, builders[2].min_part_seconds_left)
|
|
|
|
# jump forwards min_part_hours
|
|
now += min_part_hours * 3600
|
|
mock_time.return_value = now
|
|
old_builders = []
|
|
for builder in builders:
|
|
old_builder = CooperativeRingBuilder(8, 3, min_part_hours, None)
|
|
old_builder.copy_from(copy.deepcopy(builder.to_dict()))
|
|
old_builders.append(old_builder)
|
|
|
|
for builder in builders:
|
|
self.add_dev(builder)
|
|
# sanity checks: all builders are ready for rebalance
|
|
self.assertEqual(0, builders[0].min_part_seconds_left)
|
|
self.assertEqual(0, builders[1].min_part_seconds_left)
|
|
self.assertEqual(0, builders[2].min_part_seconds_left)
|
|
# ... but last_part_moves not yet updated to current epoch
|
|
if min_part_hours > 0:
|
|
self.assertEqual(0, self.num_parts_can_move(builders[0]))
|
|
self.assertEqual(0, self.num_parts_can_move(builders[1]))
|
|
self.assertEqual(0, self.num_parts_can_move(builders[2]))
|
|
|
|
with mock.patch('swift.common.ring.composite_builder.shuffle',
|
|
lambda x: x):
|
|
cb.rebalance()
|
|
|
|
rb1_parts_moved = self.get_moved_parts(builders[0], old_builders[0])
|
|
self.assertEqual(192, len(rb1_parts_moved))
|
|
self.assertEqual(self._partition_counts(builders[0]),
|
|
{0: 192, 1: 192, 2: 192, 3: 192})
|
|
|
|
rb2_parts_moved = self.get_moved_parts(builders[1], old_builders[1])
|
|
self.assertEqual(64, len(rb2_parts_moved))
|
|
counts = self._partition_counts(builders[1])
|
|
self.assertEqual(counts[3], 64)
|
|
self.assertEqual([234, 235, 235], sorted(counts.values()[:3]))
|
|
self.assertFalse(rb2_parts_moved.intersection(rb1_parts_moved))
|
|
|
|
# rb3 can't rebalance - all parts moved while rebalancing rb1 and rb2
|
|
self.assertEqual(
|
|
0, len(self.get_moved_parts(builders[2], old_builders[2])))
|
|
|
|
# jump forwards min_part_hours, all builders can move all parts again,
|
|
# so now rb2 should be able to further rebalance
|
|
now += min_part_hours * 3600
|
|
mock_time.return_value = now
|
|
old_builders = []
|
|
for builder in builders:
|
|
old_builder = CooperativeRingBuilder(8, 3, min_part_hours, None)
|
|
old_builder.copy_from(copy.deepcopy(builder.to_dict()))
|
|
old_builders.append(old_builder)
|
|
with mock.patch('swift.common.ring.composite_builder.shuffle',
|
|
lambda x: x):
|
|
cb.rebalance()
|
|
|
|
rb2_parts_moved = self.get_moved_parts(builders[1], old_builders[1])
|
|
self.assertGreater(len(rb2_parts_moved), 64)
|
|
self.assertGreater(self._partition_counts(builders[1])[3], 64)
|
|
self.assertLess(self.num_parts_can_move(builders[2]), 256)
|
|
self.assertEqual(256, self.num_parts_can_move(builders[0]))
|
|
# and rb3 should also have been able to move some parts
|
|
rb3_parts_moved = self.get_moved_parts(builders[2], old_builders[2])
|
|
self.assertGreater(len(rb3_parts_moved), 0)
|
|
self.assertFalse(rb3_parts_moved.intersection(rb2_parts_moved))
|
|
|
|
# but cobuilders will not prevent a new rb rebalancing for first time
|
|
rb4 = self._make_coop_builder(4, cb, rebalance=False,
|
|
min_part_hours=min_part_hours)
|
|
builders.append(rb4)
|
|
builder_files = []
|
|
for i, builder in enumerate(builders):
|
|
fname = os.path.join(self.tmpdir, 'builder_%s.builder' % i)
|
|
builder.save(fname)
|
|
builder_files.append(fname)
|
|
cb = CompositeRingBuilder()
|
|
builder_files, builders = cb.load_components(builder_files)
|
|
cb.rebalance()
|
|
self.assertEqual(256, len(self.get_moved_parts(builders[3], rb4)))
|
|
|
|
def test_rebalance_respects_cobuilder_part_moves(self):
|
|
self._check_rebalance_respects_cobuilder_part_moves(1)
|
|
self._check_rebalance_respects_cobuilder_part_moves(0)
|
|
|
|
@mock.patch('swift.common.ring.builder.time')
|
|
def _check_rebalance_cobuilder_states(
|
|
self, min_part_hours, mock_time):
|
|
|
|
@contextmanager
|
|
def mock_rebalance():
|
|
# wrap rebalance() in order to capture builder states before and
|
|
# after each component rebalance
|
|
orig_rebalance = RingBuilder.rebalance
|
|
# a dict mapping builder -> (list of captured builder states)
|
|
captured_builder_states = defaultdict(list)
|
|
|
|
def update_states():
|
|
for b in cb._builders:
|
|
rb = CooperativeRingBuilder(8, 3, min_part_hours, None)
|
|
rb.copy_from(copy.deepcopy(b.to_dict()))
|
|
rb._part_moved_bitmap = bytearray(b._part_moved_bitmap)
|
|
captured_builder_states[b].append(rb)
|
|
|
|
def wrap_rebalance(builder_instance):
|
|
update_states()
|
|
results = orig_rebalance(builder_instance)
|
|
update_states()
|
|
return results
|
|
|
|
with mock.patch('swift.common.ring.RingBuilder.rebalance',
|
|
wrap_rebalance):
|
|
yield captured_builder_states
|
|
|
|
mock_time.return_value = now = int(time.time())
|
|
builder_files = []
|
|
cb = CompositeRingBuilder()
|
|
for i in (1, 2, 3):
|
|
b = self._make_coop_builder(i, cb, min_part_hours=min_part_hours)
|
|
fname = os.path.join(self.tmpdir, 'builder_%s.builder' % i)
|
|
b.save(fname)
|
|
builder_files.append(fname)
|
|
builder_files, builders = cb.load_components(builder_files)
|
|
|
|
# all cobuilders can perform initial rebalance
|
|
cb.rebalance()
|
|
# jump forwards min_part_hours
|
|
now += min_part_hours * 3600
|
|
mock_time.return_value = now
|
|
for builder in builders:
|
|
self.add_dev(builder)
|
|
|
|
with mock.patch('swift.common.ring.composite_builder.shuffle',
|
|
lambda x: x):
|
|
with mock_rebalance() as captured_states:
|
|
cb.rebalance()
|
|
|
|
# sanity - state captured before and after each component rebalance
|
|
self.assertEqual(len(builders), len(captured_states))
|
|
for states in captured_states.values():
|
|
self.assertEqual(2 * len(builders), len(states))
|
|
# for each component we have a list of it's builder states
|
|
rb1s = captured_states[builders[0]]
|
|
rb2s = captured_states[builders[1]]
|
|
rb3s = captured_states[builders[2]]
|
|
|
|
# rebalancing will update epoch for all builders' last_part_moves
|
|
self.assertEqual(now, rb1s[0]._last_part_moves_epoch)
|
|
self.assertEqual(now, rb2s[0]._last_part_moves_epoch)
|
|
self.assertEqual(now, rb3s[0]._last_part_moves_epoch)
|
|
# so, in state before any component rebalance, all can now move parts
|
|
# N.B. num_parts_can_move gathers super class's (i.e. RingBuilder)
|
|
# _can_part_move so that it doesn't refer to cobuilders state.
|
|
self.assertEqual(256, self.num_parts_can_move(rb1s[0]))
|
|
self.assertEqual(256, self.num_parts_can_move(rb2s[0]))
|
|
self.assertEqual(256, self.num_parts_can_move(rb3s[0]))
|
|
|
|
# after first component has been rebalanced it has moved parts
|
|
self.assertEqual(64, self.num_parts_can_move(rb1s[1]))
|
|
self.assertEqual(256, self.num_parts_can_move(rb2s[2]))
|
|
self.assertEqual(256, self.num_parts_can_move(rb3s[2]))
|
|
|
|
rb1_parts_moved = self.get_moved_parts(rb1s[1], rb1s[0])
|
|
self.assertEqual(192, len(rb1_parts_moved))
|
|
self.assertEqual(self._partition_counts(rb1s[1]),
|
|
{0: 192, 1: 192, 2: 192, 3: 192})
|
|
|
|
# rebalancing rb2 - rb2 in isolation could potentially move all parts
|
|
# so would move 192 parts to new device, but it is constrained by rb1
|
|
# only having 64 parts that can move
|
|
rb2_parts_moved = self.get_moved_parts(rb2s[3], rb2s[2])
|
|
self.assertEqual(64, len(rb2_parts_moved))
|
|
counts = self._partition_counts(rb2s[3])
|
|
self.assertEqual(counts[3], 64)
|
|
self.assertEqual([234, 235, 235], sorted(counts.values()[:3]))
|
|
self.assertFalse(rb2_parts_moved.intersection(rb1_parts_moved))
|
|
self.assertEqual(192, self.num_parts_can_move(rb2s[3]))
|
|
self.assertEqual(64, self.num_parts_can_move(rb1s[3]))
|
|
|
|
# rb3 can't rebalance - all parts moved while rebalancing rb1 and rb2
|
|
self.assertEqual(0, len(self.get_moved_parts(rb3s[5], rb3s[0])))
|
|
|
|
def test_rebalance_cobuilder_states(self):
|
|
self._check_rebalance_cobuilder_states(1)
|
|
self._check_rebalance_cobuilder_states(0)
|
|
|
|
def _check_rebalance_cobuilders_calls(self, min_part_hours):
|
|
# verify that co-builder methods are called during one builder's
|
|
# rebalance
|
|
@contextmanager
|
|
def mock_update_last_part_moves():
|
|
# intercept calls to RingBuilder._update_last_part_moves (yes, the
|
|
# superclass method) and populate a dict mapping builder instance
|
|
# to a list of that builder's parent builder when method was called
|
|
calls = []
|
|
orig_func = RingBuilder._update_last_part_moves
|
|
|
|
def fake_update(builder):
|
|
calls.append(builder)
|
|
return orig_func(builder)
|
|
|
|
with mock.patch(
|
|
'swift.common.ring.RingBuilder._update_last_part_moves',
|
|
fake_update):
|
|
yield calls
|
|
|
|
@contextmanager
|
|
def mock_can_part_move():
|
|
# intercept calls to RingBuilder._can_part_move (yes, the
|
|
# superclass method) and populate a dict mapping builder instance
|
|
# to a list of that builder's parent builder when method was called
|
|
calls = defaultdict(list)
|
|
orig_func = RingBuilder._can_part_move
|
|
|
|
def fake_can_part_move(builder, part):
|
|
calls[builder].append(part)
|
|
return orig_func(builder, part)
|
|
with mock.patch('swift.common.ring.RingBuilder._can_part_move',
|
|
fake_can_part_move):
|
|
yield calls
|
|
|
|
cb = CompositeRingBuilder()
|
|
rb1 = self._make_coop_builder(1, cb, min_part_hours=min_part_hours)
|
|
rb2 = self._make_coop_builder(2, cb, min_part_hours=min_part_hours)
|
|
cb._builders = [rb1, rb2]
|
|
# composite rebalance updates last_part_moves before any component
|
|
# rebalance - after that expect no more updates
|
|
with mock_update_last_part_moves() as update_calls:
|
|
cb.update_last_part_moves()
|
|
self.assertEqual(sorted([rb1, rb2]), sorted(update_calls))
|
|
|
|
with mock_update_last_part_moves() as update_calls:
|
|
with mock_can_part_move() as can_part_move_calls:
|
|
rb2.rebalance()
|
|
self.assertFalse(update_calls)
|
|
# rb1 has never been rebalanced so no calls propagate from its
|
|
# can_part_move method to to its superclass _can_part_move method
|
|
self.assertEqual([rb2], can_part_move_calls.keys())
|
|
|
|
with mock_update_last_part_moves() as update_calls:
|
|
with mock_can_part_move() as can_part_move_calls:
|
|
rb1.rebalance()
|
|
self.assertFalse(update_calls)
|
|
# rb1 is being rebalanced so gets checked, and rb2 also gets checked
|
|
self.assertEqual(sorted([rb1, rb2]), sorted(can_part_move_calls))
|
|
self.assertEqual(768, len(can_part_move_calls[rb1]))
|
|
self.assertEqual(768, len(can_part_move_calls[rb2]))
|
|
|
|
def test_rebalance_cobuilders_calls(self):
|
|
self._check_rebalance_cobuilders_calls(1)
|
|
self._check_rebalance_cobuilders_calls(0)
|
|
|
|
def test_save_then_load(self):
|
|
cb = CompositeRingBuilder()
|
|
coop_rb = self._make_coop_builder(1, cb, rebalance=True)
|
|
builder_file = os.path.join(self.tmpdir, 'test.builder')
|
|
coop_rb.save(builder_file)
|
|
cb = CompositeRingBuilder()
|
|
loaded_coop_rb = CooperativeRingBuilder.load(builder_file,
|
|
parent_builder=cb)
|
|
self.assertIs(cb, loaded_coop_rb.parent_builder)
|
|
self.assertEqual(coop_rb.to_dict(), loaded_coop_rb.to_dict())
|
|
|
|
# check can be loaded as superclass
|
|
loaded_rb = RingBuilder.load(builder_file)
|
|
self.assertEqual(coop_rb.to_dict(), loaded_rb.to_dict())
|
|
|
|
# check can load a saved superclass
|
|
rb = RingBuilder(6, 3, 0)
|
|
for _ in range(3):
|
|
self.add_dev(rb, region=1)
|
|
rb.save(builder_file)
|
|
cb = CompositeRingBuilder()
|
|
loaded_coop_rb = CooperativeRingBuilder.load(builder_file,
|
|
parent_builder=cb)
|
|
self.assertIs(cb, loaded_coop_rb.parent_builder)
|
|
self.assertEqual(rb.to_dict(), loaded_coop_rb.to_dict())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|