openstack/nova
Code Issues Proposed changes

Use unique consumer_id when doing online data migration

Browse Source 
If there are multiple consumers having allocations to the same
resource provider, with different classes, it will attempt
multiple INSERTs with the same consumer_id which is not allowed
because of the database constraints.

This patch adds a simple GROUP BY in order to ensure that the
database server only provides us with unique values to avoid
trying to INSERT duplicate values.

Change-Id: I1acba5e65cd562472f29e354c6077f82844fa87d
Closes-Bug: #1798163
This commit is contained in:
Mohammed Naser 2018-10-16 19:54:40 +02:00 committed by Matt Riedemann
parent 618b47627d
commit 730936e535
3 changed files with 14 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
nova/api/openstack/placement/objects/consumer.py
View File

@ -55,6 +55,10 @@ def create_incomplete_consumers(ctx, batch_size):
sel = sa.select(cols)
sel = sel.select_from(alloc_to_consumer)
sel = sel.where(CONSUMER_TBL.c.id.is_(None))
# NOTE(mnaser): It is possible to have multiple consumers having many
# allocations to the same resource provider, which would
# make the INSERT FROM SELECT fail due to duplicates.
sel = sel.group_by(_ALLOC_TBL.c.consumer_id)
sel = sel.limit(batch_size)
target_cols = ['uuid', 'project_id', 'user_id']
ins_stmt = CONSUMER_TBL.insert().from_select(target_cols, sel)

4
nova/api/openstack/placement/objects/resource_provider.py
View File

@ -1954,6 +1954,10 @@ def _create_incomplete_consumers_for_provider(ctx, rp_id):
sa.and_(
_ALLOC_TBL.c.resource_provider_id == rp_id,
consumer_obj.CONSUMER_TBL.c.id.is_(None)))
# NOTE(mnaser): It is possible to have multiple consumers having many
# allocations to the same resource provider, which would
# make the INSERT FROM SELECT fail due to duplicates.
sel = sel.group_by(_ALLOC_TBL.c.consumer_id)
target_cols = ['uuid', 'project_id', 'user_id']
ins_stmt = consumer_obj.CONSUMER_TBL.insert().from_select(
target_cols, sel)

19
nova/tests/functional/api/openstack/placement/db/test_consumer.py
View File

@ -11,7 +11,6 @@
# under the License.
from oslo_config import cfg
from oslo_db import exception as db_exc
from oslo_utils.fixture import uuidsentinel as uuids
import sqlalchemy as sa
@ -232,20 +231,14 @@ class CreateIncompleteConsumersTestCase(base.TestCase):
# Run the online data migration to migrate one consumer. The batch size
# needs to be large enough to hit more than one consumer for this test
# where each consumer has two allocations.
# FIXME(mriedem): This should not raise a UniqueConstraint error once
# bug 1798163 is fixed.
# res = consumer_obj.create_incomplete_consumers(self.ctx, 2)
# self.assertEqual((2, 2), res)
self.assertRaises(db_exc.DBDuplicateEntry,
consumer_obj.create_incomplete_consumers,
self.ctx, 2)
res = consumer_obj.create_incomplete_consumers(self.ctx, 2)
self.assertEqual((2, 2), res)
# Migrate the rest by listing allocations on the resource provider.
rp1 = rp_obj.ResourceProvider(self.ctx, id=1)
# FIXME(mriedem): This should not raise a UniqueConstraint error once
# bug 1798163 is fixed.
self.assertRaises(db_exc.DBDuplicateEntry,
rp_obj.AllocationList.get_all_by_resource_provider,
self.ctx, rp1)
rp_obj.AllocationList.get_all_by_resource_provider(self.ctx, rp1)
self._check_incomplete_consumers(self.ctx)
res = consumer_obj.create_incomplete_consumers(self.ctx, 10)
self.assertEqual((0, 0), res)
class DeleteConsumerIfNoAllocsTestCase(tb.PlacementDbBaseTestCase):