031314b6d8
Scatter-gather utility returns a raised_exception_sentinel for all kinds of exceptions that are caught and often times there maybe situations where we may have to handle the different types of exceptions differently. To facilitate that, it might be more useful to return the Exception object itself instead of the dummy raised_exception_sentinel so that based on the result's exception type we can handle them differently. Related to blueprint handling-down-cell Change-Id: I861b223ee46b0f0a31f646a4b45f8a02410253cf
438 lines
18 KiB
Python
438 lines
18 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from contextlib import contextmanager
|
|
import copy
|
|
import datetime
|
|
import mock
|
|
from oslo_utils.fixture import uuidsentinel as uuids
|
|
|
|
from nova.compute import multi_cell_list
|
|
from nova import context
|
|
from nova import exception
|
|
from nova import objects
|
|
from nova import test
|
|
|
|
|
|
class TestUtils(test.NoDBTestCase):
|
|
def test_compare_simple(self):
|
|
dt1 = datetime.datetime(2015, 11, 5, 20, 30, 00)
|
|
dt2 = datetime.datetime(1955, 10, 25, 1, 21, 00)
|
|
|
|
inst1 = {'key0': 'foo', 'key1': 'd', 'key2': 456, 'key4': dt1}
|
|
inst2 = {'key0': 'foo', 'key1': 's', 'key2': 123, 'key4': dt2}
|
|
|
|
# Equal key0, inst == inst2
|
|
ctx = multi_cell_list.RecordSortContext(['key0'], ['asc'])
|
|
self.assertEqual(0, ctx.compare_records(inst1, inst2))
|
|
|
|
# Equal key0, inst == inst2 (direction should not matter)
|
|
ctx = multi_cell_list.RecordSortContext(['key0'], ['desc'])
|
|
self.assertEqual(0, ctx.compare_records(inst1, inst2))
|
|
|
|
# Ascending by key1, inst1 < inst2
|
|
ctx = multi_cell_list.RecordSortContext(['key1'], ['asc'])
|
|
self.assertEqual(-1, ctx.compare_records(inst1, inst2))
|
|
|
|
# Descending by key1, inst2 < inst1
|
|
ctx = multi_cell_list.RecordSortContext(['key1'], ['desc'])
|
|
self.assertEqual(1, ctx.compare_records(inst1, inst2))
|
|
|
|
# Ascending by key2, inst2 < inst1
|
|
ctx = multi_cell_list.RecordSortContext(['key2'], ['asc'])
|
|
self.assertEqual(1, ctx.compare_records(inst1, inst2))
|
|
|
|
# Descending by key2, inst1 < inst2
|
|
ctx = multi_cell_list.RecordSortContext(['key2'], ['desc'])
|
|
self.assertEqual(-1, ctx.compare_records(inst1, inst2))
|
|
|
|
# Ascending by key4, inst1 > inst2
|
|
ctx = multi_cell_list.RecordSortContext(['key4'], ['asc'])
|
|
self.assertEqual(1, ctx.compare_records(inst1, inst2))
|
|
|
|
# Descending by key4, inst1 < inst2
|
|
ctx = multi_cell_list.RecordSortContext(['key4'], ['desc'])
|
|
self.assertEqual(-1, ctx.compare_records(inst1, inst2))
|
|
|
|
def test_compare_multiple(self):
|
|
# key0 should not affect ordering, but key1 should
|
|
|
|
inst1 = {'key0': 'foo', 'key1': 'd', 'key2': 456}
|
|
inst2 = {'key0': 'foo', 'key1': 's', 'key2': 123}
|
|
|
|
# Should be equivalent to ascending by key1
|
|
ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
|
|
['asc', 'asc'])
|
|
self.assertEqual(-1, ctx.compare_records(inst1, inst2))
|
|
|
|
# Should be equivalent to descending by key1
|
|
ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
|
|
['asc', 'desc'])
|
|
self.assertEqual(1, ctx.compare_records(inst1, inst2))
|
|
|
|
def test_wrapper(self):
|
|
inst1 = {'key0': 'foo', 'key1': 'd', 'key2': 456}
|
|
inst2 = {'key0': 'foo', 'key1': 's', 'key2': 123}
|
|
|
|
ctx = context.RequestContext()
|
|
ctx.cell_uuid = uuids.cell
|
|
|
|
# Should sort by key1
|
|
sort_ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
|
|
['asc', 'asc'])
|
|
iw1 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst1)
|
|
iw2 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst2)
|
|
# Check this both ways to make sure we're comparing against -1
|
|
# and not just nonzero return from cmp()
|
|
self.assertTrue(iw1 < iw2)
|
|
self.assertFalse(iw2 < iw1)
|
|
|
|
# Should sort reverse by key1
|
|
sort_ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
|
|
['asc', 'desc'])
|
|
iw1 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst1)
|
|
iw2 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst2)
|
|
# Check this both ways to make sure we're comparing against -1
|
|
# and not just nonzero return from cmp()
|
|
self.assertTrue(iw1 > iw2)
|
|
self.assertFalse(iw2 > iw1)
|
|
|
|
# Make sure we can tell which cell a request came from
|
|
self.assertEqual(uuids.cell, iw1.cell_uuid)
|
|
|
|
def test_wrapper_sentinels(self):
|
|
inst1 = {'key0': 'foo', 'key1': 'd', 'key2': 456}
|
|
|
|
ctx = context.RequestContext()
|
|
ctx.cell_uuid = uuids.cell
|
|
|
|
sort_ctx = multi_cell_list.RecordSortContext(['key0', 'key1'],
|
|
['asc', 'asc'])
|
|
iw1 = multi_cell_list.RecordWrapper(ctx, sort_ctx, inst1)
|
|
|
|
# Wrappers with sentinels
|
|
iw2 = multi_cell_list.RecordWrapper(ctx, sort_ctx,
|
|
context.did_not_respond_sentinel)
|
|
iw3 = multi_cell_list.RecordWrapper(ctx, sort_ctx,
|
|
exception.InstanceNotFound(
|
|
instance_id='fake'))
|
|
|
|
# NOTE(danms): The sentinel wrappers always win
|
|
self.assertTrue(iw2 < iw1)
|
|
self.assertTrue(iw3 < iw1)
|
|
self.assertFalse(iw1 < iw2)
|
|
self.assertFalse(iw1 < iw3)
|
|
|
|
# NOTE(danms): Comparing two wrappers with sentinels will always return
|
|
# True for less-than because we're just naive about always favoring the
|
|
# left hand side. This is fine for our purposes but put it here to make
|
|
# it explicit.
|
|
self.assertTrue(iw2 < iw3)
|
|
self.assertTrue(iw3 < iw2)
|
|
|
|
def test_query_wrapper_success(self):
|
|
def test(ctx, data):
|
|
for thing in data:
|
|
yield thing
|
|
|
|
self.assertEqual([1, 2, 3],
|
|
list(multi_cell_list.query_wrapper(
|
|
None, test, [1, 2, 3])))
|
|
|
|
def test_query_wrapper_timeout(self):
|
|
def test(ctx):
|
|
raise exception.CellTimeout
|
|
|
|
self.assertEqual([context.did_not_respond_sentinel],
|
|
[x._db_record for x in
|
|
multi_cell_list.query_wrapper(
|
|
mock.MagicMock(), test)])
|
|
|
|
def test_query_wrapper_fail(self):
|
|
def tester(ctx):
|
|
raise test.TestingException
|
|
|
|
self.assertIsInstance(
|
|
# query_wrapper is a generator so we convert to a list and
|
|
# check the type on the first and only result
|
|
[x._db_record for x in multi_cell_list.query_wrapper(
|
|
mock.MagicMock(), tester)][0],
|
|
test.TestingException)
|
|
|
|
|
|
class TestListContext(multi_cell_list.RecordSortContext):
|
|
def compare_records(self, rec1, rec2):
|
|
return -1
|
|
|
|
|
|
class TestLister(multi_cell_list.CrossCellLister):
|
|
CONTEXT_CLS = TestListContext
|
|
|
|
def __init__(self, data, sort_keys, sort_dirs,
|
|
cells=None, batch_size=None):
|
|
self._data = data
|
|
self._count_by_cell = {}
|
|
super(TestLister, self).__init__(self.CONTEXT_CLS(sort_keys,
|
|
sort_dirs),
|
|
cells=cells, batch_size=batch_size)
|
|
|
|
@property
|
|
def marker_identifier(self):
|
|
return 'id'
|
|
|
|
def _method_called(self, ctx, method, arg):
|
|
self._count_by_cell.setdefault(ctx.cell_uuid, {})
|
|
self._count_by_cell[ctx.cell_uuid].setdefault(method, [])
|
|
self._count_by_cell[ctx.cell_uuid][method].append(arg)
|
|
|
|
def call_summary(self, method):
|
|
results = {
|
|
'total': 0,
|
|
'count_by_cell': [],
|
|
'limit_by_cell': [],
|
|
'total_by_cell': [],
|
|
'called_in_cell': [],
|
|
}
|
|
for i, cell in enumerate(self._count_by_cell):
|
|
if method not in self._count_by_cell[cell]:
|
|
continue
|
|
results['total'] += len(self._count_by_cell[cell][method])
|
|
# List of number of calls in each cell
|
|
results['count_by_cell'].append(
|
|
len(self._count_by_cell[cell][method]))
|
|
# List of limits used in calls to each cell
|
|
results['limit_by_cell'].append(
|
|
self._count_by_cell[cell][method])
|
|
try:
|
|
# List of total results fetched from each cell
|
|
results['total_by_cell'].append(sum(
|
|
self._count_by_cell[cell][method]))
|
|
except TypeError:
|
|
# Don't do this for non-integer args
|
|
pass
|
|
results['called_in_cell'].append(cell)
|
|
results['count_by_cell'].sort()
|
|
results['limit_by_cell'].sort()
|
|
results['total_by_cell'].sort()
|
|
results['called_in_cell'].sort()
|
|
return results
|
|
|
|
def get_marker_record(self, ctx, marker):
|
|
self._method_called(ctx, 'get_marker_record', marker)
|
|
# Always assume this came from the second cell
|
|
cell = self.cells[1]
|
|
return cell.uuid, self._data[0]
|
|
|
|
def get_marker_by_values(self, ctx, values):
|
|
self._method_called(ctx, 'get_marker_by_values', values)
|
|
return self._data[0]
|
|
|
|
def get_by_filters(self, ctx, filters, limit, marker, **kwargs):
|
|
self._method_called(ctx, 'get_by_filters', limit)
|
|
if 'batch_size' in kwargs:
|
|
count = min(kwargs['batch_size'], limit)
|
|
else:
|
|
count = limit
|
|
batch = self._data[:count]
|
|
self._data = self._data[count:]
|
|
return batch
|
|
|
|
|
|
@contextmanager
|
|
def target_cell_cheater(context, target_cell):
|
|
# In order to help us do accounting, we need to mimic the real
|
|
# behavior where at least cell_uuid gets set on the context, which
|
|
# doesn't happen in the simple test fixture.
|
|
context = copy.deepcopy(context)
|
|
context.cell_uuid = target_cell.uuid
|
|
yield context
|
|
|
|
|
|
@mock.patch('nova.context.target_cell', new=target_cell_cheater)
|
|
class TestBatching(test.NoDBTestCase):
|
|
def setUp(self):
|
|
super(TestBatching, self).setUp()
|
|
|
|
self._data = [{'id': 'foo-%i' % i}
|
|
for i in range(0, 1000)]
|
|
self._cells = [objects.CellMapping(uuid=getattr(uuids, 'cell%i' % i),
|
|
name='cell%i' % i)
|
|
for i in range(0, 10)]
|
|
|
|
def test_batches_not_needed(self):
|
|
lister = TestLister(self._data, [], [],
|
|
cells=self._cells, batch_size=10)
|
|
ctx = context.RequestContext()
|
|
res = list(lister.get_records_sorted(ctx, {}, 5, None))
|
|
self.assertEqual(5, len(res))
|
|
summary = lister.call_summary('get_by_filters')
|
|
# We only needed one batch per cell to hit the total,
|
|
# so we should have the same number of calls as cells
|
|
self.assertEqual(len(self._cells), summary['total'])
|
|
# One call per cell, hitting all cells
|
|
self.assertEqual(len(self._cells), len(summary['count_by_cell']))
|
|
self.assertTrue(all([
|
|
cell_count == 1 for cell_count in summary['count_by_cell']]))
|
|
|
|
def test_batches(self):
|
|
lister = TestLister(self._data, [], [],
|
|
cells=self._cells, batch_size=10)
|
|
ctx = context.RequestContext()
|
|
res = list(lister.get_records_sorted(ctx, {}, 500, None))
|
|
self.assertEqual(500, len(res))
|
|
summary = lister.call_summary('get_by_filters')
|
|
|
|
# Since we got everything from one cell (due to how things are sorting)
|
|
# we should have made 500 / 10 calls to one cell, and 1 call to
|
|
# the rest
|
|
calls_expected = [1 for cell in self._cells[1:]] + [500 / 10]
|
|
self.assertEqual(calls_expected, summary['count_by_cell'])
|
|
|
|
# Since we got everything from one cell (due to how things are sorting)
|
|
# we should have received 500 from one cell and 10 from the rest
|
|
count_expected = [10 for cell in self._cells[1:]] + [500]
|
|
self.assertEqual(count_expected, summary['total_by_cell'])
|
|
|
|
# Since we got everything from one cell (due to how things are sorting)
|
|
# we should have a bunch of calls for batches of 10, one each for
|
|
# every cell except the one that served the bulk of the requests which
|
|
# should have 500 / 10 batches of 10.
|
|
limit_expected = ([[10] for cell in self._cells[1:]] +
|
|
[[10 for i in range(0, 500 // 10)]])
|
|
self.assertEqual(limit_expected, summary['limit_by_cell'])
|
|
|
|
def test_no_batches(self):
|
|
lister = TestLister(self._data, [], [],
|
|
cells=self._cells)
|
|
ctx = context.RequestContext()
|
|
res = list(lister.get_records_sorted(ctx, {}, 50, None))
|
|
self.assertEqual(50, len(res))
|
|
summary = lister.call_summary('get_by_filters')
|
|
|
|
# Since we used no batches we should have one call per cell
|
|
calls_expected = [1 for cell in self._cells]
|
|
self.assertEqual(calls_expected, summary['count_by_cell'])
|
|
|
|
# Since we used no batches, each cell should have returned 50 results
|
|
count_expected = [50 for cell in self._cells]
|
|
self.assertEqual(count_expected, summary['total_by_cell'])
|
|
|
|
# Since we used no batches, each cell call should be for $limit
|
|
limit_expected = [[count] for count in count_expected]
|
|
self.assertEqual(limit_expected, summary['limit_by_cell'])
|
|
|
|
|
|
class FailureListContext(multi_cell_list.RecordSortContext):
|
|
def compare_records(self, rec1, rec2):
|
|
return 0
|
|
|
|
|
|
class FailureLister(TestLister):
|
|
CONTEXT_CLS = FailureListContext
|
|
|
|
def __init__(self, *a, **k):
|
|
super(FailureLister, self).__init__(*a, **k)
|
|
self._fails = {}
|
|
|
|
def set_fails(self, cell, fails):
|
|
self._fails[cell] = fails
|
|
|
|
def get_by_filters(self, ctx, *a, **k):
|
|
try:
|
|
action = self._fails[ctx.cell_uuid].pop(0)
|
|
except (IndexError, KeyError):
|
|
action = None
|
|
|
|
if action == context.did_not_respond_sentinel:
|
|
raise exception.CellTimeout
|
|
elif isinstance(action, Exception):
|
|
raise test.TestingException
|
|
else:
|
|
return super(FailureLister, self).get_by_filters(ctx, *a, **k)
|
|
|
|
|
|
@mock.patch('nova.context.target_cell', new=target_cell_cheater)
|
|
class TestBaseClass(test.NoDBTestCase):
|
|
def test_with_failing_cells(self):
|
|
data = [{'id': 'foo-%i' % i} for i in range(0, 100)]
|
|
cells = [objects.CellMapping(uuid=getattr(uuids, 'cell%i' % i),
|
|
name='cell%i' % i)
|
|
for i in range(0, 3)]
|
|
|
|
lister = FailureLister(data, [], [], cells=cells)
|
|
# Two of the cells will fail, one with timeout and one
|
|
# with an error
|
|
lister.set_fails(uuids.cell0, [context.did_not_respond_sentinel])
|
|
# Note that InstanceNotFound exception will never appear during
|
|
# instance listing, the aim is to only simulate a situation where
|
|
# there could be some type of exception arising.
|
|
lister.set_fails(uuids.cell1, exception.InstanceNotFound(
|
|
instance_id='fake'))
|
|
ctx = context.RequestContext()
|
|
result = lister.get_records_sorted(ctx, {}, 50, None, batch_size=10)
|
|
# We should still have 50 results since there are enough from the
|
|
# good cells to fill our limit.
|
|
self.assertEqual(50, len(list(result)))
|
|
# Make sure the counts line up
|
|
self.assertEqual(1, len(lister.cells_failed))
|
|
self.assertEqual(1, len(lister.cells_timed_out))
|
|
self.assertEqual(1, len(lister.cells_responded))
|
|
|
|
def test_with_failing_middle_cells(self):
|
|
data = [{'id': 'foo-%i' % i} for i in range(0, 100)]
|
|
cells = [objects.CellMapping(uuid=getattr(uuids, 'cell%i' % i),
|
|
name='cell%i' % i)
|
|
for i in range(0, 3)]
|
|
|
|
lister = FailureLister(data, [], [], cells=cells)
|
|
# One cell will succeed and then time out, one will fail immediately,
|
|
# and the last will always work
|
|
lister.set_fails(uuids.cell0, [None, context.did_not_respond_sentinel])
|
|
# Note that BuildAbortException will never appear during instance
|
|
# listing, the aim is to only simulate a situation where there could
|
|
# be some type of exception arising.
|
|
lister.set_fails(uuids.cell1, exception.BuildAbortException(
|
|
instance_uuid='fake', reason='fake'))
|
|
ctx = context.RequestContext()
|
|
result = lister.get_records_sorted(ctx, {}, 50, None,
|
|
batch_size=5)
|
|
# We should still have 50 results since there are enough from the
|
|
# good cells to fill our limit.
|
|
self.assertEqual(50, len(list(result)))
|
|
# Make sure the counts line up
|
|
self.assertEqual(1, len(lister.cells_responded))
|
|
self.assertEqual(1, len(lister.cells_failed))
|
|
self.assertEqual(1, len(lister.cells_timed_out))
|
|
|
|
def test_marker_cell_not_requeried(self):
|
|
data = [{'id': 'foo-%i' % i} for i in range(0, 100)]
|
|
cells = [objects.CellMapping(uuid=getattr(uuids, 'cell%i' % i),
|
|
name='cell%i' % i)
|
|
for i in range(0, 3)]
|
|
|
|
lister = TestLister(data, [], [], cells=cells)
|
|
ctx = context.RequestContext()
|
|
result = list(lister.get_records_sorted(ctx, {}, 10, None))
|
|
result = list(lister.get_records_sorted(ctx, {}, 10, result[-1]['id']))
|
|
|
|
# get_marker_record() is called untargeted and its result defines which
|
|
# cell we skip.
|
|
gmr_summary = lister.call_summary('get_marker_record')
|
|
self.assertEqual([None], gmr_summary['called_in_cell'])
|
|
|
|
# All cells other than the second one should have been called for
|
|
# a local marker
|
|
gmbv_summary = lister.call_summary('get_marker_by_values')
|
|
self.assertEqual(sorted([cell.uuid for cell in cells
|
|
if cell.uuid != uuids.cell1]),
|
|
gmbv_summary['called_in_cell'])
|