Use unique consumer_id when doing online data migration

If there are multiple consumers having allocations to the same
resource provider, with different classes, it will attempt
multiple INSERTs with the same consumer_id which is not allowed
because of the database constraints.

This patch adds a simple GROUP BY in order to ensure that the
database server only provides us with unique values to avoid
trying to INSERT duplicate values.

Change-Id: I1acba5e65cd562472f29e354c6077f82844fa87d
Closes-Bug: #1798163

nova/api/openstack/placement/objects/consumer.py

@@ -55,6 +55,10 @@ def create_incomplete_consumers(ctx, batch_size):
     sel = sa.select(cols)
     sel = sel.select_from(alloc_to_consumer)
     sel = sel.where(CONSUMER_TBL.c.id.is_(None))
+    # NOTE(mnaser): It is possible to have multiple consumers having many
+    #               allocations to the same resource provider, which would
+    #               make the INSERT FROM SELECT fail due to duplicates.
+    sel = sel.group_by(_ALLOC_TBL.c.consumer_id)
     sel = sel.limit(batch_size)
     target_cols = ['uuid', 'project_id', 'user_id']
     ins_stmt = CONSUMER_TBL.insert().from_select(target_cols, sel)

nova/api/openstack/placement/objects/resource_provider.py

@@ -1954,6 +1954,10 @@ def _create_incomplete_consumers_for_provider(ctx, rp_id):
             sa.and_(
                 _ALLOC_TBL.c.resource_provider_id == rp_id,
                 consumer_obj.CONSUMER_TBL.c.id.is_(None)))
+        # NOTE(mnaser): It is possible to have multiple consumers having many
+        #               allocations to the same resource provider, which would
+        #               make the INSERT FROM SELECT fail due to duplicates.
+        sel = sel.group_by(_ALLOC_TBL.c.consumer_id)
         target_cols = ['uuid', 'project_id', 'user_id']
         ins_stmt = consumer_obj.CONSUMER_TBL.insert().from_select(
             target_cols, sel)

nova/tests/functional/api/openstack/placement/db/test_consumer.py

@@ -11,7 +11,6 @@
 #    under the License.
 
 from oslo_config import cfg
-from oslo_db import exception as db_exc
 from oslo_utils.fixture import uuidsentinel as uuids
 import sqlalchemy as sa
 
@@ -232,20 +231,14 @@ class CreateIncompleteConsumersTestCase(base.TestCase):
         # Run the online data migration to migrate one consumer. The batch size
         # needs to be large enough to hit more than one consumer for this test
         # where each consumer has two allocations.
-        # FIXME(mriedem): This should not raise a UniqueConstraint error once
-        # bug 1798163 is fixed.
-        # res = consumer_obj.create_incomplete_consumers(self.ctx, 2)
-        # self.assertEqual((2, 2), res)
-        self.assertRaises(db_exc.DBDuplicateEntry,
-                          consumer_obj.create_incomplete_consumers,
-                          self.ctx, 2)
+        res = consumer_obj.create_incomplete_consumers(self.ctx, 2)
+        self.assertEqual((2, 2), res)
         # Migrate the rest by listing allocations on the resource provider.
         rp1 = rp_obj.ResourceProvider(self.ctx, id=1)
-        # FIXME(mriedem): This should not raise a UniqueConstraint error once
-        # bug 1798163 is fixed.
-        self.assertRaises(db_exc.DBDuplicateEntry,
-                          rp_obj.AllocationList.get_all_by_resource_provider,
-                          self.ctx, rp1)
+        rp_obj.AllocationList.get_all_by_resource_provider(self.ctx, rp1)
+        self._check_incomplete_consumers(self.ctx)
+        res = consumer_obj.create_incomplete_consumers(self.ctx, 10)
+        self.assertEqual((0, 0), res)
 
 
 class DeleteConsumerIfNoAllocsTestCase(tb.PlacementDbBaseTestCase):