Merge "correctly join the usage to inventory for capacity accounting"

nova/objects/resource_provider.py

@@ -10,6 +10,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from oslo_log import log as logging
 import six
 import sqlalchemy as sa
 from sqlalchemy import func
@@ -19,6 +20,7 @@ from sqlalchemy import sql
 from nova.db.sqlalchemy import api as db_api
 from nova.db.sqlalchemy import api_models as models
 from nova import exception
+from nova.i18n import _LW
 from nova import objects
 from nova.objects import base
 from nova.objects import fields
@@ -27,6 +29,8 @@ _ALLOC_TBL = models.Allocation.__table__
 _INV_TBL = models.Inventory.__table__
 _RP_TBL = models.ResourceProvider.__table__
 
+LOG = logging.getLogger(__name__)
+
 
 def _get_current_inventory_resources(conn, rp):
     """Returns a set() containing the resource class IDs for all resources
@@ -685,8 +689,10 @@ def _check_capacity_exceeded(conn, allocs):
             sql.and_(_RP_TBL.c.id == _INV_TBL.c.resource_provider_id,
                      _INV_TBL.c.resource_class_id.in_(res_classes)))
     primary_join = sql.outerjoin(inv_join, usage,
-            _INV_TBL.c.resource_provider_id == usage.c.resource_provider_id)
-
+        sql.and_(
+            _INV_TBL.c.resource_provider_id == usage.c.resource_provider_id,
+            _INV_TBL.c.resource_class_id == usage.c.resource_class_id)
+    )
     cols_in_output = [
         _RP_TBL.c.id.label('resource_provider_id'),
         _RP_TBL.c.uuid,
@@ -707,7 +713,10 @@ def _check_capacity_exceeded(conn, allocs):
     usage_map = {}
     provs_with_inv = set()
     for record in records:
-        usage_map[(record['uuid'], record['resource_class_id'])] = record
+        map_key = (record['uuid'], record['resource_class_id'])
+        if map_key in usage_map:
+            raise KeyError("%s already in usage_map, bad query" % str(map_key))
+        usage_map[map_key] = record
         provs_with_inv.add(record["uuid"])
     # Ensure that all providers have existing inventory
     missing_provs = provider_uuids - provs_with_inv
@@ -727,6 +736,14 @@ def _check_capacity_exceeded(conn, allocs):
         used = usage['used'] or 0
         capacity = (usage['total'] - usage['reserved']) * allocation_ratio
         if capacity < (used + amount_needed):
+            LOG.warning(
+                _LW("Over capacity for %(rc)s on resource provider %(rp)s. "
+                    "Needed: %(needed)s, Used: %(used)s, Capacity: %(cap)s"),
+                {'rc': fields.ResourceClass.from_index(res_class),
+                 'rp': rp_uuid,
+                 'needed': amount_needed,
+                 'used': used,
+                 'cap': capacity})
             raise exception.InvalidAllocationCapacityExceeded(
                 resource_class=fields.ResourceClass.from_index(res_class),
                 resource_provider=rp_uuid)

nova/tests/functional/db/test_resource_provider.py

@@ -574,6 +574,71 @@ class TestAllocation(ResourceProviderBaseCase):
 
 class TestAllocationListCreateDelete(ResourceProviderBaseCase):
 
+    def test_allocation_checking(self):
+        """Test that allocation check logic works with 2 resource classes on
+        one provider.
+
+        If this fails, we get a KeyError at create_all()
+        """
+
+        consumer_uuid = uuidsentinel.consumer
+        consumer_uuid2 = uuidsentinel.consumer2
+
+        # Create one resource provider with 2 classes
+        rp1_name = uuidsentinel.rp1_name
+        rp1_uuid = uuidsentinel.rp1_uuid
+        rp1_class = fields.ResourceClass.DISK_GB
+        rp1_used = 6
+
+        rp2_class = fields.ResourceClass.IPV4_ADDRESS
+        rp2_used = 2
+
+        rp1 = objects.ResourceProvider(
+            self.context, name=rp1_name, uuid=rp1_uuid)
+        rp1.create()
+
+        inv = objects.Inventory(resource_provider=rp1,
+                                resource_class=rp1_class,
+                                total=1024)
+        inv.obj_set_defaults()
+
+        inv2 = objects.Inventory(resource_provider=rp1,
+                                resource_class=rp2_class,
+                                total=255, reserved=2)
+        inv2.obj_set_defaults()
+        inv_list = objects.InventoryList(objects=[inv, inv2])
+        rp1.set_inventory(inv_list)
+
+        # create the allocations for a first consumer
+        allocation_1 = objects.Allocation(resource_provider=rp1,
+                                          consumer_id=consumer_uuid,
+                                          resource_class=rp1_class,
+                                          used=rp1_used)
+        allocation_2 = objects.Allocation(resource_provider=rp1,
+                                          consumer_id=consumer_uuid,
+                                          resource_class=rp2_class,
+                                          used=rp2_used)
+        allocation_list = objects.AllocationList(
+            self.context, objects=[allocation_1, allocation_2])
+        allocation_list.create_all()
+
+        # create the allocations for a second consumer, until we have
+        # allocations for more than one consumer in the db, then we
+        # won't actually be doing real allocation math, which triggers
+        # the sql monster.
+        allocation_1 = objects.Allocation(resource_provider=rp1,
+                                          consumer_id=consumer_uuid2,
+                                          resource_class=rp1_class,
+                                          used=rp1_used)
+        allocation_2 = objects.Allocation(resource_provider=rp1,
+                                          consumer_id=consumer_uuid2,
+                                          resource_class=rp2_class,
+                                          used=rp2_used)
+        allocation_list = objects.AllocationList(
+            self.context, objects=[allocation_1, allocation_2])
+        # If we are joining wrong, this will be a KeyError
+        allocation_list.create_all()
+
     def test_allocation_list_create(self):
         consumer_uuid = uuidsentinel.consumer