Merge "Remove incomplete consumer inline migrations"
This commit is contained in:
Zuul
Gerrit Code Review
commit
da7f598c82
placement
@ -319,86 +319,6 @@ def _get_allocations_by_consumer_uuid(ctx, consumer_uuid):
|
||||
return [dict(r) for r in ctx.session.execute(sel)]
|
||||
|
||||
|
||||
@db_api.placement_context_manager.writer.independent
|
||||
def _create_incomplete_consumers_for_provider(ctx, rp_id):
|
||||
# TODO(jaypipes): Remove in Stein after a blocker migration is added.
|
||||
"""Creates consumer record if consumer relationship between allocations ->
|
||||
consumers table is missing for any allocation on the supplied provider
|
||||
internal ID, using the "incomplete consumer" project and user CONF options.
|
||||
"""
|
||||
alloc_to_consumer = sa.outerjoin(
|
||||
_ALLOC_TBL, consumer_obj.CONSUMER_TBL,
|
||||
_ALLOC_TBL.c.consumer_id == consumer_obj.CONSUMER_TBL.c.uuid)
|
||||
sel = sa.select([_ALLOC_TBL.c.consumer_id])
|
||||
sel = sel.select_from(alloc_to_consumer)
|
||||
sel = sel.where(
|
||||
sa.and_(
|
||||
_ALLOC_TBL.c.resource_provider_id == rp_id,
|
||||
consumer_obj.CONSUMER_TBL.c.id.is_(None)))
|
||||
missing = ctx.session.execute(sel).fetchall()
|
||||
if missing:
|
||||
# Do a single INSERT for all missing consumer relationships for the
|
||||
# provider
|
||||
incomplete_proj_id = project_obj.ensure_incomplete_project(ctx)
|
||||
incomplete_user_id = user_obj.ensure_incomplete_user(ctx)
|
||||
|
||||
cols = [
|
||||
_ALLOC_TBL.c.consumer_id,
|
||||
incomplete_proj_id,
|
||||
incomplete_user_id,
|
||||
]
|
||||
sel = sa.select(cols)
|
||||
sel = sel.select_from(alloc_to_consumer)
|
||||
sel = sel.where(
|
||||
sa.and_(
|
||||
_ALLOC_TBL.c.resource_provider_id == rp_id,
|
||||
consumer_obj.CONSUMER_TBL.c.id.is_(None)))
|
||||
# NOTE(mnaser): It is possible to have multiple consumers having many
|
||||
# allocations to the same resource provider, which would
|
||||
# make the INSERT FROM SELECT fail due to duplicates.
|
||||
sel = sel.group_by(_ALLOC_TBL.c.consumer_id)
|
||||
target_cols = ['uuid', 'project_id', 'user_id']
|
||||
ins_stmt = consumer_obj.CONSUMER_TBL.insert().from_select(
|
||||
target_cols, sel)
|
||||
res = ctx.session.execute(ins_stmt)
|
||||
if res.rowcount > 0:
|
||||
LOG.info("Online data migration to fix incomplete consumers "
|
||||
"for resource provider %s has been run. Migrated %d "
|
||||
"incomplete consumer records on the fly.", rp_id,
|
||||
res.rowcount)
|
||||
|
||||
|
||||
@db_api.placement_context_manager.writer.independent
|
||||
def _create_incomplete_consumer(ctx, consumer_id):
|
||||
# TODO(jaypipes): Remove in Stein after a blocker migration is added.
|
||||
"""Creates consumer record if consumer relationship between allocations ->
|
||||
consumers table is missing for the supplied consumer UUID, using the
|
||||
"incomplete consumer" project and user CONF options.
|
||||
"""
|
||||
alloc_to_consumer = sa.outerjoin(
|
||||
_ALLOC_TBL, consumer_obj.CONSUMER_TBL,
|
||||
_ALLOC_TBL.c.consumer_id == consumer_obj.CONSUMER_TBL.c.uuid)
|
||||
sel = sa.select([_ALLOC_TBL.c.consumer_id])
|
||||
sel = sel.select_from(alloc_to_consumer)
|
||||
sel = sel.where(
|
||||
sa.and_(
|
||||
_ALLOC_TBL.c.consumer_id == consumer_id,
|
||||
consumer_obj.CONSUMER_TBL.c.id.is_(None)))
|
||||
missing = ctx.session.execute(sel).fetchall()
|
||||
if missing:
|
||||
incomplete_proj_id = project_obj.ensure_incomplete_project(ctx)
|
||||
incomplete_user_id = user_obj.ensure_incomplete_user(ctx)
|
||||
|
||||
ins_stmt = consumer_obj.CONSUMER_TBL.insert().values(
|
||||
uuid=consumer_id, project_id=incomplete_proj_id,
|
||||
user_id=incomplete_user_id)
|
||||
res = ctx.session.execute(ins_stmt)
|
||||
if res.rowcount > 0:
|
||||
LOG.info("Online data migration to fix incomplete consumers "
|
||||
"for consumer %s has been run. Migrated %d incomplete "
|
||||
"consumer records on the fly.", consumer_id, res.rowcount)
|
||||
|
||||
|
||||
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
|
||||
@db_api.placement_context_manager.writer
|
||||
def _set_allocations(context, allocs):
|
||||
@ -487,7 +407,6 @@ def _set_allocations(context, allocs):
|
||||
|
||||
|
||||
def get_all_by_resource_provider(context, rp):
|
||||
_create_incomplete_consumers_for_provider(context, rp.id)
|
||||
db_allocs = _get_allocations_by_provider_id(context, rp.id)
|
||||
# Build up a list of Allocation objects, setting the Allocation object
|
||||
# fields to the same-named database record field we got from
|
||||
@ -519,7 +438,6 @@ def get_all_by_resource_provider(context, rp):
|
||||
|
||||
|
||||
def get_all_by_consumer_id(context, consumer_id):
|
||||
_create_incomplete_consumer(context, consumer_id)
|
||||
db_allocs = _get_allocations_by_consumer_uuid(context, consumer_id)
|
||||
|
||||
if not db_allocs:
|
||||
|
||||
@ -192,60 +192,14 @@ class CreateIncompleteConsumersTestCase(
|
||||
# consumer_obj.create_incomplete_consumers() with a batch size of 1.
|
||||
# This should mean there will be two allocation records still remaining
|
||||
# with a missing consumer record (since we create 3 total to begin
|
||||
# with). We then query the allocations table directly to grab that
|
||||
# consumer UUID in the allocations table that doesn't refer to a
|
||||
# consumer table record and call
|
||||
# alloc_obj.get_all_by_consumer_id() with that consumer UUID. This
|
||||
# should create the remaining missing consumer record "inline" in the
|
||||
# alloc_obj.get_all_by_consumer_id() method.
|
||||
# After that happens, there should still be a single allocation record
|
||||
# that is missing a relation to the consumers table. We call the
|
||||
# alloc_obj.get_all_by_resource_provider() method and verify that
|
||||
# method cleans up the remaining incomplete consumers relationship.
|
||||
# with).
|
||||
res = consumer_obj.create_incomplete_consumers(self.ctx, 1)
|
||||
self.assertEqual((1, 1), res)
|
||||
|
||||
# Grab the consumer UUID for the allocation record with a
|
||||
# still-incomplete consumer record.
|
||||
# Confirm there are still 2 incomplete allocations after one
|
||||
# interation of the migration.
|
||||
res = _get_allocs_with_no_consumer_relationship(self.ctx)
|
||||
self.assertEqual(2, len(res))
|
||||
still_missing = res[0][0]
|
||||
alloc_obj.get_all_by_consumer_id(self.ctx, still_missing)
|
||||
|
||||
# There should still be a single missing consumer relationship. Let's
|
||||
# grab that and call alloc_obj.get_all_by_resource_provider()
|
||||
# which should clean that last one up for us.
|
||||
res = _get_allocs_with_no_consumer_relationship(self.ctx)
|
||||
self.assertEqual(1, len(res))
|
||||
still_missing = res[0][0]
|
||||
rp1 = rp_obj.ResourceProvider(self.ctx, id=1)
|
||||
alloc_obj.get_all_by_resource_provider(self.ctx, rp1)
|
||||
|
||||
# get_all_by_resource_provider() should have auto-completed the still
|
||||
# missing consumer record and _check_incomplete_consumers() should
|
||||
# assert correctly that there are no more incomplete consumer records.
|
||||
self._check_incomplete_consumers(self.ctx)
|
||||
res = consumer_obj.create_incomplete_consumers(self.ctx, 10)
|
||||
self.assertEqual((0, 0), res)
|
||||
|
||||
def test_create_incomplete_consumers_multiple_allocs_per_consumer(self):
|
||||
"""Tests that missing consumer records are created when listing
|
||||
allocations against a resource provider or running the online data
|
||||
migration routine when the consumers have multiple allocations on the
|
||||
same provider.
|
||||
"""
|
||||
self._create_incomplete_allocations(self.ctx, num_of_consumer_allocs=2)
|
||||
# Run the online data migration to migrate one consumer. The batch size
|
||||
# needs to be large enough to hit more than one consumer for this test
|
||||
# where each consumer has two allocations.
|
||||
res = consumer_obj.create_incomplete_consumers(self.ctx, 2)
|
||||
self.assertEqual((2, 2), res)
|
||||
# Migrate the rest by listing allocations on the resource provider.
|
||||
rp1 = rp_obj.ResourceProvider(self.ctx, id=1)
|
||||
alloc_obj.get_all_by_resource_provider(self.ctx, rp1)
|
||||
self._check_incomplete_consumers(self.ctx)
|
||||
res = consumer_obj.create_incomplete_consumers(self.ctx, 10)
|
||||
self.assertEqual((0, 0), res)
|
||||
|
||||
|
||||
class DeleteConsumerIfNoAllocsTestCase(tb.PlacementDbBaseTestCase):
|
||||
|
||||
@ -68,13 +68,10 @@ class TestAllocationListNoDB(base.TestCase):
|
||||
super(TestAllocationListNoDB, self).setUp()
|
||||
base.fake_ensure_cache(self.context)
|
||||
|
||||
@mock.patch('placement.objects.allocation.'
|
||||
'_create_incomplete_consumers_for_provider')
|
||||
@mock.patch('placement.objects.allocation.'
|
||||
'_get_allocations_by_provider_id',
|
||||
return_value=[_ALLOCATION_DB])
|
||||
def test_get_all_by_resource_provider(
|
||||
self, mock_get_allocations_from_db, mock_create_consumers):
|
||||
def test_get_all_by_resource_provider(self, mock_get_allocations_from_db):
|
||||
rp = rp_obj.ResourceProvider(self.context,
|
||||
id=_RESOURCE_PROVIDER_ID,
|
||||
uuid=uuids.resource_provider)
|
||||
@ -88,22 +85,15 @@ class TestAllocationListNoDB(base.TestCase):
|
||||
allocations[0].created_at)
|
||||
self.assertEqual(_ALLOCATION_DB['updated_at'],
|
||||
allocations[0].updated_at)
|
||||
mock_create_consumers.assert_called_once_with(
|
||||
self.context, _RESOURCE_PROVIDER_ID)
|
||||
|
||||
@mock.patch('placement.objects.allocation.'
|
||||
'_create_incomplete_consumer')
|
||||
@mock.patch('placement.objects.allocation.'
|
||||
'_get_allocations_by_consumer_uuid',
|
||||
return_value=[_ALLOCATION_BY_CONSUMER_DB])
|
||||
def test_get_all_by_consumer_id(self, mock_get_allocations_from_db,
|
||||
mock_create_consumer):
|
||||
def test_get_all_by_consumer_id(self, mock_get_allocations_from_db):
|
||||
allocations = alloc_obj.get_all_by_consumer_id(
|
||||
self.context, uuids.consumer)
|
||||
|
||||
self.assertEqual(1, len(allocations))
|
||||
mock_create_consumer.assert_called_once_with(self.context,
|
||||
uuids.consumer)
|
||||
mock_get_allocations_from_db.assert_called_once_with(self.context,
|
||||
uuids.consumer)
|
||||
self.assertEqual(_ALLOCATION_BY_CONSUMER_DB['used'],
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user