From 6bd7e8a958f53e2288b41a27528a008e3cfb2802 Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Tue, 16 Oct 2018 19:54:40 +0200 Subject: [PATCH] Use unique consumer_id when doing online data migration If there are multiple consumers having allocations to the same resource provider, with different classes, it will attempt multiple INSERTs with the same consumer_id which is not allowed because of the database constraints. This patch adds a simple GROUP BY in order to ensure that the database server only provides us with unique values to avoid trying to INSERT duplicate values. Conflicts: nova/tests/functional/api/openstack/placement/db/test_consumer.py NOTE(mriedem): The conflict is due to not having change I7f5f08691ca3f73073c66c29dddb996fb2c2b266 in Rocky. Change-Id: I1acba5e65cd562472f29e354c6077f82844fa87d Closes-Bug: #1798163 (cherry picked from commit 730936e535e67127c76d4f27649a16d8cf05efc9) --- .../openstack/placement/objects/consumer.py | 4 ++++ .../placement/objects/resource_provider.py | 4 ++++ .../openstack/placement/db/test_consumer.py | 19 ++++++------------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/nova/api/openstack/placement/objects/consumer.py b/nova/api/openstack/placement/objects/consumer.py index 75bd93dfbdb4..9d88a83adc4d 100644 --- a/nova/api/openstack/placement/objects/consumer.py +++ b/nova/api/openstack/placement/objects/consumer.py @@ -55,6 +55,10 @@ def create_incomplete_consumers(ctx, batch_size): sel = sa.select(cols) sel = sel.select_from(alloc_to_consumer) sel = sel.where(CONSUMER_TBL.c.id.is_(None)) + # NOTE(mnaser): It is possible to have multiple consumers having many + # allocations to the same resource provider, which would + # make the INSERT FROM SELECT fail due to duplicates. + sel = sel.group_by(_ALLOC_TBL.c.consumer_id) sel = sel.limit(batch_size) target_cols = ['uuid', 'project_id', 'user_id'] ins_stmt = CONSUMER_TBL.insert().from_select(target_cols, sel) diff --git a/nova/api/openstack/placement/objects/resource_provider.py b/nova/api/openstack/placement/objects/resource_provider.py index 3a876f9f16f1..e6a778a4ead8 100644 --- a/nova/api/openstack/placement/objects/resource_provider.py +++ b/nova/api/openstack/placement/objects/resource_provider.py @@ -1917,6 +1917,10 @@ def _create_incomplete_consumers_for_provider(ctx, rp_id): sa.and_( _ALLOC_TBL.c.resource_provider_id == rp_id, consumer_obj.CONSUMER_TBL.c.id.is_(None))) + # NOTE(mnaser): It is possible to have multiple consumers having many + # allocations to the same resource provider, which would + # make the INSERT FROM SELECT fail due to duplicates. + sel = sel.group_by(_ALLOC_TBL.c.consumer_id) target_cols = ['uuid', 'project_id', 'user_id'] ins_stmt = consumer_obj.CONSUMER_TBL.insert().from_select( target_cols, sel) diff --git a/nova/tests/functional/api/openstack/placement/db/test_consumer.py b/nova/tests/functional/api/openstack/placement/db/test_consumer.py index 9055b650a5e2..dbc65f4dd5e7 100644 --- a/nova/tests/functional/api/openstack/placement/db/test_consumer.py +++ b/nova/tests/functional/api/openstack/placement/db/test_consumer.py @@ -11,7 +11,6 @@ # under the License. from oslo_config import cfg -from oslo_db import exception as db_exc import sqlalchemy as sa from nova.api.openstack.placement import db_api @@ -231,20 +230,14 @@ class CreateIncompleteConsumersTestCase(base.TestCase): # Run the online data migration to migrate one consumer. The batch size # needs to be large enough to hit more than one consumer for this test # where each consumer has two allocations. - # FIXME(mriedem): This should not raise a UniqueConstraint error once - # bug 1798163 is fixed. - # res = consumer_obj.create_incomplete_consumers(self.ctx, 2) - # self.assertEqual((2, 2), res) - self.assertRaises(db_exc.DBDuplicateEntry, - consumer_obj.create_incomplete_consumers, - self.ctx, 2) + res = consumer_obj.create_incomplete_consumers(self.ctx, 2) + self.assertEqual((2, 2), res) # Migrate the rest by listing allocations on the resource provider. rp1 = rp_obj.ResourceProvider(self.ctx, id=1) - # FIXME(mriedem): This should not raise a UniqueConstraint error once - # bug 1798163 is fixed. - self.assertRaises(db_exc.DBDuplicateEntry, - rp_obj.AllocationList.get_all_by_resource_provider, - self.ctx, rp1) + rp_obj.AllocationList.get_all_by_resource_provider(self.ctx, rp1) + self._check_incomplete_consumers(self.ctx) + res = consumer_obj.create_incomplete_consumers(self.ctx, 10) + self.assertEqual((0, 0), res) class DeleteConsumerIfNoAllocsTestCase(tb.PlacementDbBaseTestCase):