Add DB support for resource usage tracking

This patch introduces database support for tracking Neutron
resource usage data. A single DB model class tracks usage
info for all neutron resources.

The patch also provides a simple API for managing resource
usage info, as well as unit tests providing coverage for
this API.

This patch also makes a slight change to the ContextBase
class, adding the ability to explicitly set is_advsvc at
initialization time. While this probably makes no difference
for practical use of the context class, it simplifies
development of DB-only unit tests.

Related-Blueprint: better-quotas

Change-Id: I62100551b89103a21555dcc45e84195c05e89800

neutron/context.py

@@ -39,7 +39,8 @@ class ContextBase(oslo_context.RequestContext):
     @removals.removed_kwarg('read_deleted')
     def __init__(self, user_id, tenant_id, is_admin=None, roles=None,
                  timestamp=None, request_id=None, tenant_name=None,
-                 user_name=None, overwrite=True, auth_token=None, **kwargs):
+                 user_name=None, overwrite=True, auth_token=None,
+                 is_advsvc=None, **kwargs):
         """Object initialization.
 
         :param overwrite: Set to False to ensure that the greenthread local
@@ -60,7 +61,9 @@ class ContextBase(oslo_context.RequestContext):
             timestamp = datetime.datetime.utcnow()
         self.timestamp = timestamp
         self.roles = roles or []
-        self.is_advsvc = self.is_admin or policy.check_is_advsvc(self)
+        self.is_advsvc = is_advsvc
+        if self.is_advsvc is None:
+            self.is_advsvc = self.is_admin or policy.check_is_advsvc(self)
         if self.is_admin is None:
             self.is_admin = policy.check_is_admin(self)
 

neutron/db/migration/alembic_migrations/versions/HEADS

@@ -1,3 +1,3 @@
 2a16083502f3
-8675309a5c4f
+45f955889773
 kilo

neutron/db/migration/alembic_migrations/versions/liberty/expand/45f955889773_quota_usage.py

@@ -0,0 +1,45 @@
+# Copyright 2015 OpenStack Foundation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""quota_usage
+
+Revision ID: 45f955889773
+Revises: 8675309a5c4f
+Create Date: 2015-04-17 08:09:37.611546
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '45f955889773'
+down_revision = '8675309a5c4f'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import sql
+
+
+def upgrade():
+    op.create_table(
+        'quotausages',
+        sa.Column('tenant_id', sa.String(length=255),
+                  nullable=False, primary_key=True, index=True),
+        sa.Column('resource', sa.String(length=255),
+                  nullable=False, primary_key=True, index=True),
+        sa.Column('dirty', sa.Boolean(), nullable=False,
+                  server_default=sql.false()),
+        sa.Column('in_use', sa.Integer(), nullable=False,
+                  server_default='0'),
+        sa.Column('reserved', sa.Integer(), nullable=False,
+                  server_default='0'))

neutron/db/migration/models/head.py

@@ -41,7 +41,7 @@ from neutron.db import model_base
 from neutron.db import models_v2  # noqa
 from neutron.db import portbindings_db  # noqa
 from neutron.db import portsecurity_db  # noqa
-from neutron.db import quota_db  # noqa
+from neutron.db.quota import models  # noqa
 from neutron.db import rbac_db_models  # noqa
 from neutron.db import securitygroups_db  # noqa
 from neutron.db import servicetype_db  # noqa

neutron/db/quota/api.py

@@ -0,0 +1,159 @@
+# Copyright (c) 2015 OpenStack Foundation.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import collections
+
+from neutron.db import common_db_mixin as common_db_api
+from neutron.db.quota import models as quota_models
+
+
+class QuotaUsageInfo(collections.namedtuple(
+    'QuotaUsageInfo', ['resource', 'tenant_id', 'used', 'reserved', 'dirty'])):
+
+    @property
+    def total(self):
+        """Total resource usage (reserved and used)."""
+        return self.reserved + self.used
+
+
+def get_quota_usage_by_resource_and_tenant(context, resource, tenant_id,
+                                           lock_for_update=False):
+    """Return usage info for a given resource and tenant.
+
+    :param context: Request context
+    :param resource: Name of the resource
+    :param tenant_id: Tenant identifier
+    :param lock_for_update: if True sets a write-intent lock on the query
+    :returns: a QuotaUsageInfo instance
+    """
+
+    query = common_db_api.model_query(context, quota_models.QuotaUsage)
+    query = query.filter_by(resource=resource, tenant_id=tenant_id)
+
+    if lock_for_update:
+        query = query.with_lockmode('update')
+
+    result = query.first()
+    if not result:
+        return
+    return QuotaUsageInfo(result.resource,
+                          result.tenant_id,
+                          result.in_use,
+                          result.reserved,
+                          result.dirty)
+
+
+def get_quota_usage_by_resource(context, resource):
+    query = common_db_api.model_query(context, quota_models.QuotaUsage)
+    query = query.filter_by(resource=resource)
+    return [QuotaUsageInfo(item.resource,
+                           item.tenant_id,
+                           item.in_use,
+                           item.reserved,
+                           item.dirty) for item in query]
+
+
+def get_quota_usage_by_tenant_id(context, tenant_id):
+    query = common_db_api.model_query(context, quota_models.QuotaUsage)
+    query = query.filter_by(tenant_id=tenant_id)
+    return [QuotaUsageInfo(item.resource,
+                           item.tenant_id,
+                           item.in_use,
+                           item.reserved,
+                           item.dirty) for item in query]
+
+
+def set_quota_usage(context, resource, tenant_id,
+                    in_use=None, reserved=None, delta=False):
+    """Set resource quota usage.
+
+    :param context: instance of neutron context with db session
+    :param resource: name of the resource for which usage is being set
+    :param tenant_id: identifier of the tenant for which quota usage is
+                      being set
+    :param in_use: integer specifying the new quantity of used resources,
+                   or a delta to apply to current used resource
+    :param reserved: integer specifying the new quantity of reserved resources,
+                     or a delta to apply to current reserved resources
+    :param delta: Specififies whether in_use or reserved are absolute numbers
+                  or deltas (default to False)
+    """
+    query = common_db_api.model_query(context, quota_models.QuotaUsage)
+    query = query.filter_by(resource=resource).filter_by(tenant_id=tenant_id)
+    usage_data = query.first()
+    with context.session.begin(subtransactions=True):
+        if not usage_data:
+            # Must create entry
+            usage_data = quota_models.QuotaUsage(
+                resource=resource,
+                tenant_id=tenant_id)
+            context.session.add(usage_data)
+        # Perform explicit comparison with None as 0 is a valid value
+        if in_use is not None:
+            if delta:
+                in_use = usage_data.in_use + in_use
+            usage_data.in_use = in_use
+        if reserved is not None:
+            if delta:
+                reserved = usage_data.reserved + reserved
+            usage_data.reserved = reserved
+        # After an explicit update the dirty bit should always be reset
+        usage_data.dirty = False
+    return QuotaUsageInfo(usage_data.resource,
+                          usage_data.tenant_id,
+                          usage_data.in_use,
+                          usage_data.reserved,
+                          usage_data.dirty)
+
+
+def set_quota_usage_dirty(context, resource, tenant_id, dirty=True):
+    """Set quota usage dirty bit for a given resource and tenant.
+
+    :param resource: a resource for which quota usage if tracked
+    :param tenant_id: tenant identifier
+    :param dirty: the desired value for the dirty bit (defaults to True)
+    :returns: 1 if the quota usage data were updated, 0 otherwise.
+    """
+    query = common_db_api.model_query(context, quota_models.QuotaUsage)
+    query = query.filter_by(resource=resource).filter_by(tenant_id=tenant_id)
+    return query.update({'dirty': dirty})
+
+
+def set_resources_quota_usage_dirty(context, resources, tenant_id, dirty=True):
+    """Set quota usage dirty bit for a given tenant and multiple resources.
+
+    :param resources: list of resource for which the dirty bit is going
+                      to be set
+    :param tenant_id: tenant identifier
+    :param dirty: the desired value for the dirty bit (defaults to True)
+    :returns: the number of records for which the bit was actually set.
+    """
+    query = common_db_api.model_query(context, quota_models.QuotaUsage)
+    query = query.filter_by(tenant_id=tenant_id)
+    if resources:
+        query = query.filter(quota_models.QuotaUsage.resource.in_(resources))
+    # synchronize_session=False needed because of the IN condition
+    return query.update({'dirty': dirty}, synchronize_session=False)
+
+
+def set_all_quota_usage_dirty(context, resource, dirty=True):
+    """Set the dirty bit on quota usage for all tenants.
+
+    :param resource: the resource for which the dirty bit should be set
+    :returns: the number of tenants for which the dirty bit was
+              actually updated
+    """
+    query = common_db_api.model_query(context, quota_models.QuotaUsage)
+    query = query.filter_by(resource=resource)
+    return query.update({'dirty': dirty})

neutron/db/quota/models.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2015 OpenStack Foundation.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import sqlalchemy as sa
+from sqlalchemy import sql
+
+from neutron.db import model_base
+from neutron.db import models_v2
+
+
+class Quota(model_base.BASEV2, models_v2.HasId, models_v2.HasTenant):
+    """Represent a single quota override for a tenant.
+
+    If there is no row for a given tenant id and resource, then the
+    default for the deployment is used.
+    """
+    resource = sa.Column(sa.String(255))
+    limit = sa.Column(sa.Integer)
+
+
+class QuotaUsage(model_base.BASEV2):
+    """Represents the current usage for a given resource."""
+
+    resource = sa.Column(sa.String(255), nullable=False,
+                         primary_key=True, index=True)
+    tenant_id = sa.Column(sa.String(255), nullable=False,
+                          primary_key=True, index=True)
+    dirty = sa.Column(sa.Boolean, nullable=False, server_default=sql.false())
+
+    in_use = sa.Column(sa.Integer, nullable=False,
+                       server_default="0")
+    reserved = sa.Column(sa.Integer, nullable=False,
+                         server_default="0")

neutron/db/quota_db.py

@@ -13,21 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import sqlalchemy as sa
-
 from neutron.common import exceptions
-from neutron.db import model_base
-from neutron.db import models_v2
-
-
-class Quota(model_base.BASEV2, models_v2.HasId, models_v2.HasTenant):
-    """Represent a single quota override for a tenant.
-
-    If there is no row for a given tenant id and resource, then the
-    default for the deployment is used.
-    """
-    resource = sa.Column(sa.String(255))
-    limit = sa.Column(sa.Integer)
+from neutron.db.quota import models as quota_models
 
 
 class DbQuotaDriver(object):
@@ -53,7 +40,8 @@ class DbQuotaDriver(object):
                             for key, resource in resources.items())
 
         # update with tenant specific limits
-        q_qry = context.session.query(Quota).filter_by(tenant_id=tenant_id)
+        q_qry = context.session.query(quota_models.Quota).filter_by(
+            tenant_id=tenant_id)
         tenant_quota.update((q['resource'], q['limit']) for q in q_qry)
 
         return tenant_quota
@@ -65,7 +53,7 @@ class DbQuotaDriver(object):
         Atfer deletion, this tenant will use default quota values in conf.
         """
         with context.session.begin():
-            tenant_quotas = context.session.query(Quota)
+            tenant_quotas = context.session.query(quota_models.Quota)
             tenant_quotas = tenant_quotas.filter_by(tenant_id=tenant_id)
             tenant_quotas.delete()
 
@@ -83,7 +71,7 @@ class DbQuotaDriver(object):
 
         all_tenant_quotas = {}
 
-        for quota in context.session.query(Quota):
+        for quota in context.session.query(quota_models.Quota):
             tenant_id = quota['tenant_id']
 
             # avoid setdefault() because only want to copy when actually req'd
@@ -100,15 +88,15 @@ class DbQuotaDriver(object):
     @staticmethod
     def update_quota_limit(context, tenant_id, resource, limit):
         with context.session.begin():
-            tenant_quota = context.session.query(Quota).filter_by(
+            tenant_quota = context.session.query(quota_models.Quota).filter_by(
                 tenant_id=tenant_id, resource=resource).first()
 
             if tenant_quota:
                 tenant_quota.update({'limit': limit})
             else:
-                tenant_quota = Quota(tenant_id=tenant_id,
-                                     resource=resource,
-                                     limit=limit)
+                tenant_quota = quota_models.Quota(tenant_id=tenant_id,
+                                                  resource=resource,
+                                                  limit=limit)
                 context.session.add(tenant_quota)
 
     def _get_quotas(self, context, tenant_id, resources):

neutron/tests/unit/db/quota/test_api.py

@@ -0,0 +1,229 @@
+# Copyright (c) 2015 OpenStack Foundation.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron import context
+from neutron.db.quota import api as quota_api
+from neutron.tests.unit import testlib_api
+
+
+class TestQuotaDbApi(testlib_api.SqlTestCaseLight):
+
+    def _set_context(self):
+        self.tenant_id = 'Higuain'
+        self.context = context.Context('Gonzalo', self.tenant_id,
+                                       is_admin=False, is_advsvc=False)
+
+    def _create_quota_usage(self, resource, used, reserved, tenant_id=None):
+        tenant_id = tenant_id or self.tenant_id
+        return quota_api.set_quota_usage(
+            self.context, resource, tenant_id,
+            in_use=used, reserved=reserved)
+
+    def _verify_quota_usage(self, usage_info,
+                            expected_resource=None,
+                            expected_used=None,
+                            expected_reserved=None,
+                            expected_dirty=None):
+        self.assertEqual(self.tenant_id, usage_info.tenant_id)
+        if expected_resource:
+            self.assertEqual(expected_resource, usage_info.resource)
+        if expected_dirty is not None:
+                self.assertEqual(expected_dirty, usage_info.dirty)
+        if expected_used is not None:
+            self.assertEqual(expected_used, usage_info.used)
+        if expected_reserved is not None:
+            self.assertEqual(expected_reserved, usage_info.reserved)
+        if expected_used is not None and expected_reserved is not None:
+            self.assertEqual(expected_used + expected_reserved,
+                             usage_info.total)
+
+    def setUp(self):
+        super(TestQuotaDbApi, self).setUp()
+        self._set_context()
+
+    def test_create_quota_usage(self):
+        usage_info = self._create_quota_usage('goals', 26, 10)
+        self._verify_quota_usage(usage_info,
+                                 expected_resource='goals',
+                                 expected_used=26,
+                                 expected_reserved=10)
+
+    def test_update_quota_usage(self):
+        self._create_quota_usage('goals', 26, 10)
+        # Higuain scores a double
+        usage_info_1 = quota_api.set_quota_usage(
+            self.context, 'goals', self.tenant_id,
+            in_use=28)
+        self._verify_quota_usage(usage_info_1,
+                                 expected_used=28,
+                                 expected_reserved=10)
+        usage_info_2 = quota_api.set_quota_usage(
+            self.context, 'goals', self.tenant_id,
+            reserved=8)
+        self._verify_quota_usage(usage_info_2,
+                                 expected_used=28,
+                                 expected_reserved=8)
+
+    def test_update_quota_usage_with_deltas(self):
+        self._create_quota_usage('goals', 26, 10)
+        # Higuain scores a double
+        usage_info_1 = quota_api.set_quota_usage(
+            self.context, 'goals', self.tenant_id,
+            in_use=2, delta=True)
+        self._verify_quota_usage(usage_info_1,
+                                 expected_used=28,
+                                 expected_reserved=10)
+        usage_info_2 = quota_api.set_quota_usage(
+            self.context, 'goals', self.tenant_id,
+            reserved=-2, delta=True)
+        self._verify_quota_usage(usage_info_2,
+                                 expected_used=28,
+                                 expected_reserved=8)
+
+    def test_set_quota_usage_dirty(self):
+        self._create_quota_usage('goals', 26, 10)
+        # Higuain needs a shower after the match
+        self.assertEqual(1, quota_api.set_quota_usage_dirty(
+            self.context, 'goals', self.tenant_id))
+        usage_info = quota_api.get_quota_usage_by_resource_and_tenant(
+            self.context, 'goals', self.tenant_id)
+        self._verify_quota_usage(usage_info,
+                                 expected_dirty=True)
+        # Higuain is clean now
+        self.assertEqual(1, quota_api.set_quota_usage_dirty(
+            self.context, 'goals', self.tenant_id, dirty=False))
+        usage_info = quota_api.get_quota_usage_by_resource_and_tenant(
+            self.context, 'goals', self.tenant_id)
+        self._verify_quota_usage(usage_info,
+                                 expected_dirty=False)
+
+    def test_set_dirty_non_existing_quota_usage(self):
+        self.assertEqual(0, quota_api.set_quota_usage_dirty(
+            self.context, 'meh', self.tenant_id))
+
+    def test_set_resources_quota_usage_dirty(self):
+        self._create_quota_usage('goals', 26, 10)
+        self._create_quota_usage('assists', 11, 5)
+        self._create_quota_usage('bookings', 3, 1)
+        self.assertEqual(2, quota_api.set_resources_quota_usage_dirty(
+            self.context, ['goals', 'bookings'], self.tenant_id))
+        usage_info_goals = quota_api.get_quota_usage_by_resource_and_tenant(
+            self.context, 'goals', self.tenant_id)
+        usage_info_assists = quota_api.get_quota_usage_by_resource_and_tenant(
+            self.context, 'assists', self.tenant_id)
+        usage_info_bookings = quota_api.get_quota_usage_by_resource_and_tenant(
+            self.context, 'bookings', self.tenant_id)
+        self._verify_quota_usage(usage_info_goals, expected_dirty=True)
+        self._verify_quota_usage(usage_info_assists, expected_dirty=False)
+        self._verify_quota_usage(usage_info_bookings, expected_dirty=True)
+
+    def test_set_resources_quota_usage_dirty_with_empty_list(self):
+        self._create_quota_usage('goals', 26, 10)
+        self._create_quota_usage('assists', 11, 5)
+        self._create_quota_usage('bookings', 3, 1)
+        # Expect all the resources for the tenant to be set dirty
+        self.assertEqual(3, quota_api.set_resources_quota_usage_dirty(
+            self.context, [], self.tenant_id))
+        usage_info_goals = quota_api.get_quota_usage_by_resource_and_tenant(
+            self.context, 'goals', self.tenant_id)
+        usage_info_assists = quota_api.get_quota_usage_by_resource_and_tenant(
+            self.context, 'assists', self.tenant_id)
+        usage_info_bookings = quota_api.get_quota_usage_by_resource_and_tenant(
+            self.context, 'bookings', self.tenant_id)
+        self._verify_quota_usage(usage_info_goals, expected_dirty=True)
+        self._verify_quota_usage(usage_info_assists, expected_dirty=True)
+        self._verify_quota_usage(usage_info_bookings, expected_dirty=True)
+
+        # Higuain is clean now
+        self.assertEqual(1, quota_api.set_quota_usage_dirty(
+            self.context, 'goals', self.tenant_id, dirty=False))
+        usage_info = quota_api.get_quota_usage_by_resource_and_tenant(
+            self.context, 'goals', self.tenant_id)
+        self._verify_quota_usage(usage_info,
+                                 expected_dirty=False)
+
+    def _test_set_all_quota_usage_dirty(self, expected):
+        self._create_quota_usage('goals', 26, 10)
+        self._create_quota_usage('goals', 12, 6, tenant_id='Callejon')
+        self.assertEqual(expected, quota_api.set_all_quota_usage_dirty(
+            self.context, 'goals'))
+
+    def test_set_all_quota_usage_dirty(self):
+        # All goal scorers need a shower after the match, but since this is not
+        # admin context we can clean only one
+        self._test_set_all_quota_usage_dirty(expected=1)
+
+    def test_get_quota_usage_by_tenant(self):
+        self._create_quota_usage('goals', 26, 10)
+        self._create_quota_usage('assists', 11, 5)
+        # Create a resource for a different tenant
+        self._create_quota_usage('mehs', 99, 99, tenant_id='buffon')
+        usage_infos = quota_api.get_quota_usage_by_tenant_id(
+            self.context, self.tenant_id)
+
+        self.assertEqual(2, len(usage_infos))
+        resources = [info.resource for info in usage_infos]
+        self.assertIn('goals', resources)
+        self.assertIn('assists', resources)
+
+    def test_get_quota_usage_by_resource(self):
+        self._create_quota_usage('goals', 26, 10)
+        self._create_quota_usage('assists', 11, 5)
+        self._create_quota_usage('goals', 12, 6, tenant_id='Callejon')
+        usage_infos = quota_api.get_quota_usage_by_resource(
+            self.context, 'goals')
+        # Only 1 result expected in tenant context
+        self.assertEqual(1, len(usage_infos))
+        self._verify_quota_usage(usage_infos[0],
+                                 expected_resource='goals',
+                                 expected_used=26,
+                                 expected_reserved=10)
+
+    def test_get_quota_usage_by_tenant_and_resource(self):
+        self._create_quota_usage('goals', 26, 10)
+        usage_info = quota_api.get_quota_usage_by_resource_and_tenant(
+            self.context, 'goals', self.tenant_id)
+        self._verify_quota_usage(usage_info,
+                                 expected_resource='goals',
+                                 expected_used=26,
+                                 expected_reserved=10)
+
+    def test_get_non_existing_quota_usage_returns_none(self):
+        self.assertIsNone(quota_api.get_quota_usage_by_resource_and_tenant(
+            self.context, 'goals', self.tenant_id))
+
+
+class TestQuotaDbApiAdminContext(TestQuotaDbApi):
+
+    def _set_context(self):
+        self.tenant_id = 'Higuain'
+        self.context = context.Context('Gonzalo', self.tenant_id,
+                                       is_admin=True, is_advsvc=True,
+                                       load_admin_roles=False)
+
+    def test_get_quota_usage_by_resource(self):
+        self._create_quota_usage('goals', 26, 10)
+        self._create_quota_usage('assists', 11, 5)
+        self._create_quota_usage('goals', 12, 6, tenant_id='Callejon')
+        usage_infos = quota_api.get_quota_usage_by_resource(
+            self.context, 'goals')
+        # 2 results expected in admin context
+        self.assertEqual(2, len(usage_infos))
+        for usage_info in usage_infos:
+            self.assertEqual('goals', usage_info.resource)
+
+    def test_set_all_quota_usage_dirty(self):
+        # All goal scorers need a shower after the match, and with admin
+        # context we should be able to clean all of them
+        self._test_set_all_quota_usage_dirty(expected=2)