438 lines
18 KiB
Python
438 lines
18 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from contextlib import contextmanager
|
|
import copy
|
|
import datetime
|
|
import mock
|
|
from oslo_utils.fixture import uuidsentinel as uuids
|
|
|
|
from nova.compute import multi_cell_list
|
|
from nova import context
|
|
from nova import exception
|
|
from nova import objects
|
|
from nova import test
|
|
|
|
|
|
class TestUtils(test.NoDBTestCase):
|
|
def test_compare_simple(self):
|
|
dt1 = datetime.datetime(2015, 11, 5, 20, 30, 00)
|
|
dt2 = datetime.datetime(1955, 10, 25, 1, 21, 00)
|
|
|
|
inst1 = {'key0': 'foo', 'key1': 'd', 'key2': 456, 'key4': dt1}
|
|
inst2 = {'key0': 'foo', 'key1': 's', 'key2': 123, 'key4': dt2}
|
|
|
|
# Equal key0, inst == inst2
|
|
ctx = multi_cell_list.RecordSortContext(['key0'], ['asc'])
|
|
self.assertEqual(0, ctx.compare_records(inst1, inst2))
|
|
|
|
# Equal key0, inst == inst2 (direction should not matter)
|
|
ctx = multi_cell_list.RecordSortContext(['key0'], ['desc'])
|
|
self.assertEqual(0, ctx.compare_records(inst1, inst2))
|
|
|
|
# Ascending by key1, inst1 < inst2
|
|
ctx = multi_cell_list.RecordSortContext(['key1'], ['asc'])
|
|
self.assertEqual(-1, ctx.compare_records(inst1, inst2))
|
|
|
|
# Descending by key1, inst2 < inst1
|
|
ctx = multi_cell_list.RecordSortContext(['key1'], ['desc'])
|
|
self.assertEqual(1, ctx.compare_records(inst1, inst2))
|
|
|
|
# Ascending by key2, inst2 < inst1
|
|
ctx = multi_cell_list.RecordSortContext(['key2'], ['asc'])
|
|
self.assertEqual(1, ctx.compare_records(inst1, inst2))
|
|
|
|
# Descending by key2, inst1 < inst2
|
|
ctx = multi_cell_list.RecordSortContext(['key2'], ['desc'])
|
|
self.assertEqual(-1, ctx.compare_records(inst1, inst2))
|
|
|
|
# Ascending by key4, inst1 > inst2
|
|
ctx = multi_cell_list.RecordSortContext(['key4'], ['asc'])
|
|
self.assertEqual(1, ctx.compare_records(inst1, inst2))
|
|
|
|
# Descending by key4, inst1 < inst2
|
|
ctx = multi_cell_list.RecordSortContext(['key4'], ['desc'])
|
|
self.assertEqual(-1, ctx.compare_records(inst1, inst2))
|
|
|
|
def test_compare_multiple(self):
|
|
# key0 should not affect ordering, but key1 should
|
|
|
|
inst1 = {'key0': 'foo', 'key1': 'd', 'key2': 456}
|
|
inst2 = {'key0': 'foo', 'key1': 's', 'key2': 123}
|
|
|
|
# Should be equivalent to ascending by key1
|
|
ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
|
|
['asc', 'asc'])
|
|
self.assertEqual(-1, ctx.compare_records(inst1, inst2))
|
|
|
|
# Should be equivalent to descending by key1
|
|
ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
|
|
['asc', 'desc'])
|
|
self.assertEqual(1, ctx.compare_records(inst1, inst2))
|
|
|
|
def test_wrapper(self):
|
|
inst1 = {'key0': 'foo', 'key1': 'd', 'key2': 456}
|
|
inst2 = {'key0': 'foo', 'key1': 's', 'key2': 123}
|
|
|
|
ctx = context.RequestContext()
|
|
ctx.cell_uuid = uuids.cell
|
|
|
|
# Should sort by key1
|
|
sort_ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
|
|
['asc', 'asc'])
|
|
iw1 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst1)
|
|
iw2 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst2)
|
|
# Check this both ways to make sure we're comparing against -1
|
|
# and not just nonzero return from cmp()
|
|
self.assertTrue(iw1 < iw2)
|
|
self.assertFalse(iw2 < iw1)
|
|
|
|
# Should sort reverse by key1
|
|
sort_ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
|
|
['asc', 'desc'])
|
|
iw1 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst1)
|
|
iw2 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst2)
|
|
# Check this both ways to make sure we're comparing against -1
|
|
# and not just nonzero return from cmp()
|
|
self.assertTrue(iw1 > iw2)
|
|
self.assertFalse(iw2 > iw1)
|
|
|
|
# Make sure we can tell which cell a request came from
|
|
self.assertEqual(uuids.cell, iw1.cell_uuid)
|
|
|
|
def test_wrapper_sentinels(self):
|
|
inst1 = {'key0': 'foo', 'key1': 'd', 'key2': 456}
|
|
|
|
ctx = context.RequestContext()
|
|
ctx.cell_uuid = uuids.cell
|
|
|
|
sort_ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
|
|
['asc', 'asc'])
|
|
iw1 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst1)
|
|
|
|
# Wrappers with sentinels
|
|
iw2 = multi_cell_list.RecordWrapper(ctx, sort_ctx,
|
|
context.did_not_respond_sentinel)
|
|
iw3 = multi_cell_list.RecordWrapper(ctx, sort_ctx,
|
|
exception.InstanceNotFound(
|
|
instance_id='fake'))
|
|
|
|
# NOTE(danms): The sentinel wrappers always win
|
|
self.assertTrue(iw2 < iw1)
|
|
self.assertTrue(iw3 < iw1)
|
|
self.assertFalse(iw1 < iw2)
|
|
self.assertFalse(iw1 < iw3)
|
|
|
|
# NOTE(danms): Comparing two wrappers with sentinels will always return
|
|
# True for less-than because we're just naive about always favoring the
|
|
# left hand side. This is fine for our purposes but put it here to make
|
|
# it explicit.
|
|
self.assertTrue(iw2 < iw3)
|
|
self.assertTrue(iw3 < iw2)
|
|
|
|
def test_query_wrapper_success(self):
|
|
def test(ctx, data):
|
|
for thing in data:
|
|
yield thing
|
|
|
|
self.assertEqual([1, 2, 3],
|
|
list(multi_cell_list.query_wrapper(
|
|
None, test, [1, 2, 3])))
|
|
|
|
def test_query_wrapper_timeout(self):
|
|
def test(ctx):
|
|
raise exception.CellTimeout
|
|
|
|
self.assertEqual([context.did_not_respond_sentinel],
|
|
[x._db_record for x in
|
|
multi_cell_list.query_wrapper(
|
|
mock.MagicMock(), test)])
|
|
|
|
def test_query_wrapper_fail(self):
|
|
def tester(ctx):
|
|
raise test.TestingException
|
|
|
|
self.assertIsInstance(
|
|
# query_wrapper is a generator so we convert to a list and
|
|
# check the type on the first and only result
|
|
[x._db_record for x in multi_cell_list.query_wrapper(
|
|
mock.MagicMock(), tester)][0],
|
|
test.TestingException)
|
|
|
|
|
|
class TestListContext(multi_cell_list.RecordSortContext):
|
|
def compare_records(self, rec1, rec2):
|
|
return -1
|
|
|
|
|
|
class TestLister(multi_cell_list.CrossCellLister):
|
|
CONTEXT_CLS = TestListContext
|
|
|
|
def __init__(self, data, sort_keys, sort_dirs,
|
|
cells=None, batch_size=None):
|
|
self._data = data
|
|
self._count_by_cell = {}
|
|
super(TestLister, self).__init__(self.CONTEXT_CLS(sort_keys,
|
|
sort_dirs),
|
|
cells=cells, batch_size=batch_size)
|
|
|
|
@property
|
|
def marker_identifier(self):
|
|
return 'id'
|
|
|
|
def _method_called(self, ctx, method, arg):
|
|
self._count_by_cell.setdefault(ctx.cell_uuid, {})
|
|
self._count_by_cell[ctx.cell_uuid].setdefault(method, [])
|
|
self._count_by_cell[ctx.cell_uuid][method].append(arg)
|
|
|
|
def call_summary(self, method):
|
|
results = {
|
|
'total': 0,
|
|
'count_by_cell': [],
|
|
'limit_by_cell': [],
|
|
'total_by_cell': [],
|
|
'called_in_cell': [],
|
|
}
|
|
for i, cell in enumerate(self._count_by_cell):
|
|
if method not in self._count_by_cell[cell]:
|
|
continue
|
|
results['total'] += len(self._count_by_cell[cell][method])
|
|
# List of number of calls in each cell
|
|
results['count_by_cell'].append(
|
|
len(self._count_by_cell[cell][method]))
|
|
# List of limits used in calls to each cell
|
|
results['limit_by_cell'].append(
|
|
self._count_by_cell[cell][method])
|
|
try:
|
|
# List of total results fetched from each cell
|
|
results['total_by_cell'].append(sum(
|
|
self._count_by_cell[cell][method]))
|
|
except TypeError:
|
|
# Don't do this for non-integer args
|
|
pass
|
|
results['called_in_cell'].append(cell)
|
|
results['count_by_cell'].sort()
|
|
results['limit_by_cell'].sort()
|
|
results['total_by_cell'].sort()
|
|
results['called_in_cell'].sort()
|
|
return results
|
|
|
|
def get_marker_record(self, ctx, marker):
|
|
self._method_called(ctx, 'get_marker_record', marker)
|
|
# Always assume this came from the second cell
|
|
cell = self.cells[1]
|
|
return cell.uuid, self._data[0]
|
|
|
|
def get_marker_by_values(self, ctx, values):
|
|
self._method_called(ctx, 'get_marker_by_values', values)
|
|
return self._data[0]
|
|
|
|
def get_by_filters(self, ctx, filters, limit, marker, **kwargs):
|
|
self._method_called(ctx, 'get_by_filters', limit)
|
|
if 'batch_size' in kwargs:
|
|
count = min(kwargs['batch_size'], limit)
|
|
else:
|
|
count = limit
|
|
batch = self._data[:count]
|
|
self._data = self._data[count:]
|
|
return batch
|
|
|
|
|
|
@contextmanager
|
|
def target_cell_cheater(context, target_cell):
|
|
# In order to help us do accounting, we need to mimic the real
|
|
# behavior where at least cell_uuid gets set on the context, which
|
|
# doesn't happen in the simple test fixture.
|
|
context = copy.deepcopy(context)
|
|
context.cell_uuid = target_cell.uuid
|
|
yield context
|
|
|
|
|
|
@mock.patch('nova.context.target_cell', new=target_cell_cheater)
|
|
class TestBatching(test.NoDBTestCase):
|
|
def setUp(self):
|
|
super(TestBatching, self).setUp()
|
|
|
|
self._data = [{'id': 'foo-%i' % i}
|
|
for i in range(0, 1000)]
|
|
self._cells = [objects.CellMapping(uuid=getattr(uuids, 'cell%i' % i),
|
|
name='cell%i' % i)
|
|
for i in range(0, 10)]
|
|
|
|
def test_batches_not_needed(self):
|
|
lister = TestLister(self._data, [], [],
|
|
cells=self._cells, batch_size=10)
|
|
ctx = context.RequestContext()
|
|
res = list(lister.get_records_sorted(ctx, {}, 5, None))
|
|
self.assertEqual(5, len(res))
|
|
summary = lister.call_summary('get_by_filters')
|
|
# We only needed one batch per cell to hit the total,
|
|
# so we should have the same number of calls as cells
|
|
self.assertEqual(len(self._cells), summary['total'])
|
|
# One call per cell, hitting all cells
|
|
self.assertEqual(len(self._cells), len(summary['count_by_cell']))
|
|
self.assertTrue(all([
|
|
cell_count == 1 for cell_count in summary['count_by_cell']]))
|
|
|
|
def test_batches(self):
|
|
lister = TestLister(self._data, [], [],
|
|
cells=self._cells, batch_size=10)
|
|
ctx = context.RequestContext()
|
|
res = list(lister.get_records_sorted(ctx, {}, 500, None))
|
|
self.assertEqual(500, len(res))
|
|
summary = lister.call_summary('get_by_filters')
|
|
|
|
# Since we got everything from one cell (due to how things are sorting)
|
|
# we should have made 500 / 10 calls to one cell, and 1 call to
|
|
# the rest
|
|
calls_expected = [1 for cell in self._cells[1:]] + [500 / 10]
|
|
self.assertEqual(calls_expected, summary['count_by_cell'])
|
|
|
|
# Since we got everything from one cell (due to how things are sorting)
|
|
# we should have received 500 from one cell and 10 from the rest
|
|
count_expected = [10 for cell in self._cells[1:]] + [500]
|
|
self.assertEqual(count_expected, summary['total_by_cell'])
|
|
|
|
# Since we got everything from one cell (due to how things are sorting)
|
|
# we should have a bunch of calls for batches of 10, one each for
|
|
# every cell except the one that served the bulk of the requests which
|
|
# should have 500 / 10 batches of 10.
|
|
limit_expected = ([[10] for cell in self._cells[1:]] +
|
|
[[10 for i in range(0, 500 // 10)]])
|
|
self.assertEqual(limit_expected, summary['limit_by_cell'])
|
|
|
|
def test_no_batches(self):
|
|
lister = TestLister(self._data, [], [],
|
|
cells=self._cells)
|
|
ctx = context.RequestContext()
|
|
res = list(lister.get_records_sorted(ctx, {}, 50, None))
|
|
self.assertEqual(50, len(res))
|
|
summary = lister.call_summary('get_by_filters')
|
|
|
|
# Since we used no batches we should have one call per cell
|
|
calls_expected = [1 for cell in self._cells]
|
|
self.assertEqual(calls_expected, summary['count_by_cell'])
|
|
|
|
# Since we used no batches, each cell should have returned 50 results
|
|
count_expected = [50 for cell in self._cells]
|
|
self.assertEqual(count_expected, summary['total_by_cell'])
|
|
|
|
# Since we used no batches, each cell call should be for $limit
|
|
limit_expected = [[count] for count in count_expected]
|
|
self.assertEqual(limit_expected, summary['limit_by_cell'])
|
|
|
|
|
|
class FailureListContext(multi_cell_list.RecordSortContext):
|
|
def compare_records(self, rec1, rec2):
|
|
return 0
|
|
|
|
|
|
class FailureLister(TestLister):
|
|
CONTEXT_CLS = FailureListContext
|
|
|
|
def __init__(self, *a, **k):
|
|
super(FailureLister, self).__init__(*a, **k)
|
|
self._fails = {}
|
|
|
|
def set_fails(self, cell, fails):
|
|
self._fails[cell] = fails
|
|
|
|
def get_by_filters(self, ctx, *a, **k):
|
|
try:
|
|
action = self._fails[ctx.cell_uuid].pop(0)
|
|
except (IndexError, KeyError):
|
|
action = None
|
|
|
|
if action == context.did_not_respond_sentinel:
|
|
raise exception.CellTimeout
|
|
elif isinstance(action, Exception):
|
|
raise test.TestingException
|
|
else:
|
|
return super(FailureLister, self).get_by_filters(ctx, *a, **k)
|
|
|
|
|
|
@mock.patch('nova.context.target_cell', new=target_cell_cheater)
|
|
class TestBaseClass(test.NoDBTestCase):
|
|
def test_with_failing_cells(self):
|
|
data = [{'id': 'foo-%i' % i} for i in range(0, 100)]
|
|
cells = [objects.CellMapping(uuid=getattr(uuids, 'cell%i' % i),
|
|
name='cell%i' % i)
|
|
for i in range(0, 3)]
|
|
|
|
lister = FailureLister(data, [], [], cells=cells)
|
|
# Two of the cells will fail, one with timeout and one
|
|
# with an error
|
|
lister.set_fails(uuids.cell0, [context.did_not_respond_sentinel])
|
|
# Note that InstanceNotFound exception will never appear during
|
|
# instance listing, the aim is to only simulate a situation where
|
|
# there could be some type of exception arising.
|
|
lister.set_fails(uuids.cell1, exception.InstanceNotFound(
|
|
instance_id='fake'))
|
|
ctx = context.RequestContext()
|
|
result = lister.get_records_sorted(ctx, {}, 50, None, batch_size=10)
|
|
# We should still have 50 results since there are enough from the
|
|
# good cells to fill our limit.
|
|
self.assertEqual(50, len(list(result)))
|
|
# Make sure the counts line up
|
|
self.assertEqual(1, len(lister.cells_failed))
|
|
self.assertEqual(1, len(lister.cells_timed_out))
|
|
self.assertEqual(1, len(lister.cells_responded))
|
|
|
|
def test_with_failing_middle_cells(self):
|
|
data = [{'id': 'foo-%i' % i} for i in range(0, 100)]
|
|
cells = [objects.CellMapping(uuid=getattr(uuids, 'cell%i' % i),
|
|
name='cell%i' % i)
|
|
for i in range(0, 3)]
|
|
|
|
lister = FailureLister(data, [], [], cells=cells)
|
|
# One cell will succeed and then time out, one will fail immediately,
|
|
# and the last will always work
|
|
lister.set_fails(uuids.cell0, [None, context.did_not_respond_sentinel])
|
|
# Note that BuildAbortException will never appear during instance
|
|
# listing, the aim is to only simulate a situation where there could
|
|
# be some type of exception arising.
|
|
lister.set_fails(uuids.cell1, exception.BuildAbortException(
|
|
instance_uuid='fake', reason='fake'))
|
|
ctx = context.RequestContext()
|
|
result = lister.get_records_sorted(ctx, {}, 50, None,
|
|
batch_size=5)
|
|
# We should still have 50 results since there are enough from the
|
|
# good cells to fill our limit.
|
|
self.assertEqual(50, len(list(result)))
|
|
# Make sure the counts line up
|
|
self.assertEqual(1, len(lister.cells_responded))
|
|
self.assertEqual(1, len(lister.cells_failed))
|
|
self.assertEqual(1, len(lister.cells_timed_out))
|
|
|
|
def test_marker_cell_not_requeried(self):
|
|
data = [{'id': 'foo-%i' % i} for i in range(0, 100)]
|
|
cells = [objects.CellMapping(uuid=getattr(uuids, 'cell%i' % i),
|
|
name='cell%i' % i)
|
|
for i in range(0, 3)]
|
|
|
|
lister = TestLister(data, [], [], cells=cells)
|
|
ctx = context.RequestContext()
|
|
result = list(lister.get_records_sorted(ctx, {}, 10, None))
|
|
result = list(lister.get_records_sorted(ctx, {}, 10, result[-1]['id']))
|
|
|
|
# get_marker_record() is called untargeted and its result defines which
|
|
# cell we skip.
|
|
gmr_summary = lister.call_summary('get_marker_record')
|
|
self.assertEqual([None], gmr_summary['called_in_cell'])
|
|
|
|
# All cells other than the second one should have been called for
|
|
# a local marker
|
|
gmbv_summary = lister.call_summary('get_marker_by_values')
|
|
self.assertEqual(sorted([cell.uuid for cell in cells
|
|
if cell.uuid != uuids.cell1]),
|
|
gmbv_summary['called_in_cell'])
|