openstack
/
neutron
Code Issues Proposed changes

Add DB support for resource usage tracking 

This patch introduces database support for tracking Neutron
resource usage data. A single DB model class tracks usage
info for all neutron resources.

The patch also provides a simple API for managing resource
usage info, as well as unit tests providing coverage for
this API.

This patch also makes a slight change to the ContextBase
class, adding the ability to explicitly set is_advsvc at
initialization time. While this probably makes no difference
for practical use of the context class, it simplifies
development of DB-only unit tests.

Related-Blueprint: better-quotas

Change-Id: I62100551b89103a21555dcc45e84195c05e89800
This commit is contained in:
Salvatore Orlando 2015-05-19 09:49:26 -07:00
parent e95bc6f5be
commit b3d4851f63
10 changed files with 493 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
neutron/context.py
View File

@ -39,7 +39,8 @@ class ContextBase(oslo_context.RequestContext):
@removals.removed_kwarg('read_deleted')
def __init__(self, user_id, tenant_id, is_admin=None, roles=None,
timestamp=None, request_id=None, tenant_name=None,
user_name=None, overwrite=True, auth_token=None, **kwargs):
user_name=None, overwrite=True, auth_token=None,
is_advsvc=None, **kwargs):
"""Object initialization.
:param overwrite: Set to False to ensure that the greenthread local
@ -60,7 +61,9 @@ class ContextBase(oslo_context.RequestContext):
timestamp = datetime.datetime.utcnow()
self.timestamp = timestamp
self.roles = roles or []
self.is_advsvc = self.is_admin or policy.check_is_advsvc(self)
self.is_advsvc = is_advsvc
if self.is_advsvc is None:
self.is_advsvc = self.is_admin or policy.check_is_advsvc(self)
if self.is_admin is None:
self.is_admin = policy.check_is_admin(self)

2
neutron/db/migration/alembic_migrations/versions/HEADS
View File

@ -1,3 +1,3 @@
2a16083502f3
8675309a5c4f
45f955889773
kilo

45
neutron/db/migration/alembic_migrations/versions/liberty/expand/45f955889773_quota_usage.py Normal file
View File

@ -0,0 +1,45 @@
# Copyright 2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
"""quota_usage
Revision ID: 45f955889773
Revises: 8675309a5c4f
Create Date: 2015-04-17 08:09:37.611546
"""
# revision identifiers, used by Alembic.
revision = '45f955889773'
down_revision = '8675309a5c4f'
from alembic import op
import sqlalchemy as sa
from sqlalchemy import sql
def upgrade():
op.create_table(
'quotausages',
sa.Column('tenant_id', sa.String(length=255),
nullable=False, primary_key=True, index=True),
sa.Column('resource', sa.String(length=255),
nullable=False, primary_key=True, index=True),
sa.Column('dirty', sa.Boolean(), nullable=False,
server_default=sql.false()),
sa.Column('in_use', sa.Integer(), nullable=False,
server_default='0'),
sa.Column('reserved', sa.Integer(), nullable=False,
server_default='0'))

2
neutron/db/migration/models/head.py
View File

@ -41,7 +41,7 @@ from neutron.db import model_base
from neutron.db import models_v2 # noqa
from neutron.db import portbindings_db # noqa
from neutron.db import portsecurity_db # noqa
from neutron.db import quota_db # noqa
from neutron.db.quota import models # noqa
from neutron.db import rbac_db_models # noqa
from neutron.db import securitygroups_db # noqa
from neutron.db import servicetype_db # noqa

0
neutron/db/quota/__init__.py Normal file
View File

159
neutron/db/quota/api.py Normal file
View File

@ -0,0 +1,159 @@
# Copyright (c) 2015 OpenStack Foundation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
from neutron.db import common_db_mixin as common_db_api
from neutron.db.quota import models as quota_models
class QuotaUsageInfo(collections.namedtuple(
'QuotaUsageInfo', ['resource', 'tenant_id', 'used', 'reserved', 'dirty'])):
@property
def total(self):
"""Total resource usage (reserved and used)."""
return self.reserved + self.used
def get_quota_usage_by_resource_and_tenant(context, resource, tenant_id,
lock_for_update=False):
"""Return usage info for a given resource and tenant.
:param context: Request context
:param resource: Name of the resource
:param tenant_id: Tenant identifier
:param lock_for_update: if True sets a write-intent lock on the query
:returns: a QuotaUsageInfo instance
"""
query = common_db_api.model_query(context, quota_models.QuotaUsage)
query = query.filter_by(resource=resource, tenant_id=tenant_id)
if lock_for_update:
query = query.with_lockmode('update')
result = query.first()
if not result:
return
return QuotaUsageInfo(result.resource,
result.tenant_id,
result.in_use,
result.reserved,
result.dirty)
def get_quota_usage_by_resource(context, resource):
query = common_db_api.model_query(context, quota_models.QuotaUsage)
query = query.filter_by(resource=resource)
return [QuotaUsageInfo(item.resource,
item.tenant_id,
item.in_use,
item.reserved,
item.dirty) for item in query]
def get_quota_usage_by_tenant_id(context, tenant_id):
query = common_db_api.model_query(context, quota_models.QuotaUsage)
query = query.filter_by(tenant_id=tenant_id)
return [QuotaUsageInfo(item.resource,
item.tenant_id,
item.in_use,
item.reserved,
item.dirty) for item in query]
def set_quota_usage(context, resource, tenant_id,
in_use=None, reserved=None, delta=False):
"""Set resource quota usage.
:param context: instance of neutron context with db session
:param resource: name of the resource for which usage is being set
:param tenant_id: identifier of the tenant for which quota usage is
being set
:param in_use: integer specifying the new quantity of used resources,
or a delta to apply to current used resource
:param reserved: integer specifying the new quantity of reserved resources,
or a delta to apply to current reserved resources
:param delta: Specififies whether in_use or reserved are absolute numbers
or deltas (default to False)
"""
query = common_db_api.model_query(context, quota_models.QuotaUsage)
query = query.filter_by(resource=resource).filter_by(tenant_id=tenant_id)
usage_data = query.first()
with context.session.begin(subtransactions=True):
if not usage_data:
# Must create entry
usage_data = quota_models.QuotaUsage(
resource=resource,
tenant_id=tenant_id)
context.session.add(usage_data)
# Perform explicit comparison with None as 0 is a valid value
if in_use is not None:
if delta:
in_use = usage_data.in_use + in_use
usage_data.in_use = in_use
if reserved is not None:
if delta:
reserved = usage_data.reserved + reserved
usage_data.reserved = reserved
# After an explicit update the dirty bit should always be reset
usage_data.dirty = False
return QuotaUsageInfo(usage_data.resource,
usage_data.tenant_id,
usage_data.in_use,
usage_data.reserved,
usage_data.dirty)
def set_quota_usage_dirty(context, resource, tenant_id, dirty=True):
"""Set quota usage dirty bit for a given resource and tenant.
:param resource: a resource for which quota usage if tracked
:param tenant_id: tenant identifier
:param dirty: the desired value for the dirty bit (defaults to True)
:returns: 1 if the quota usage data were updated, 0 otherwise.
"""
query = common_db_api.model_query(context, quota_models.QuotaUsage)
query = query.filter_by(resource=resource).filter_by(tenant_id=tenant_id)
return query.update({'dirty': dirty})
def set_resources_quota_usage_dirty(context, resources, tenant_id, dirty=True):
"""Set quota usage dirty bit for a given tenant and multiple resources.
:param resources: list of resource for which the dirty bit is going
to be set
:param tenant_id: tenant identifier
:param dirty: the desired value for the dirty bit (defaults to True)
:returns: the number of records for which the bit was actually set.
"""
query = common_db_api.model_query(context, quota_models.QuotaUsage)
query = query.filter_by(tenant_id=tenant_id)
if resources:
query = query.filter(quota_models.QuotaUsage.resource.in_(resources))
# synchronize_session=False needed because of the IN condition
return query.update({'dirty': dirty}, synchronize_session=False)
def set_all_quota_usage_dirty(context, resource, dirty=True):
"""Set the dirty bit on quota usage for all tenants.
:param resource: the resource for which the dirty bit should be set
:returns: the number of tenants for which the dirty bit was
actually updated
"""
query = common_db_api.model_query(context, quota_models.QuotaUsage)
query = query.filter_by(resource=resource)
return query.update({'dirty': dirty})

44
neutron/db/quota/models.py Normal file
View File

@ -0,0 +1,44 @@
# Copyright (c) 2015 OpenStack Foundation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sa
from sqlalchemy import sql
from neutron.db import model_base
from neutron.db import models_v2
class Quota(model_base.BASEV2, models_v2.HasId, models_v2.HasTenant):
"""Represent a single quota override for a tenant.
If there is no row for a given tenant id and resource, then the
default for the deployment is used.
"""
resource = sa.Column(sa.String(255))
limit = sa.Column(sa.Integer)
class QuotaUsage(model_base.BASEV2):
"""Represents the current usage for a given resource."""
resource = sa.Column(sa.String(255), nullable=False,
primary_key=True, index=True)
tenant_id = sa.Column(sa.String(255), nullable=False,
primary_key=True, index=True)
dirty = sa.Column(sa.Boolean, nullable=False, server_default=sql.false())
in_use = sa.Column(sa.Integer, nullable=False,
server_default="0")
reserved = sa.Column(sa.Integer, nullable=False,
server_default="0")

30
neutron/db/quota_db.py
View File

@ -13,21 +13,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sa
from neutron.common import exceptions
from neutron.db import model_base
from neutron.db import models_v2
class Quota(model_base.BASEV2, models_v2.HasId, models_v2.HasTenant):
"""Represent a single quota override for a tenant.
If there is no row for a given tenant id and resource, then the
default for the deployment is used.
"""
resource = sa.Column(sa.String(255))
limit = sa.Column(sa.Integer)
from neutron.db.quota import models as quota_models
class DbQuotaDriver(object):
@ -53,7 +40,8 @@ class DbQuotaDriver(object):
for key, resource in resources.items())
# update with tenant specific limits
q_qry = context.session.query(Quota).filter_by(tenant_id=tenant_id)
q_qry = context.session.query(quota_models.Quota).filter_by(
tenant_id=tenant_id)
tenant_quota.update((q['resource'], q['limit']) for q in q_qry)
return tenant_quota
@ -65,7 +53,7 @@ class DbQuotaDriver(object):
Atfer deletion, this tenant will use default quota values in conf.
"""
with context.session.begin():
tenant_quotas = context.session.query(Quota)
tenant_quotas = context.session.query(quota_models.Quota)
tenant_quotas = tenant_quotas.filter_by(tenant_id=tenant_id)
tenant_quotas.delete()
@ -83,7 +71,7 @@ class DbQuotaDriver(object):
all_tenant_quotas = {}
for quota in context.session.query(Quota):
for quota in context.session.query(quota_models.Quota):
tenant_id = quota['tenant_id']
# avoid setdefault() because only want to copy when actually req'd
@ -100,15 +88,15 @@ class DbQuotaDriver(object):
@staticmethod
def update_quota_limit(context, tenant_id, resource, limit):
with context.session.begin():
tenant_quota = context.session.query(Quota).filter_by(
tenant_quota = context.session.query(quota_models.Quota).filter_by(
tenant_id=tenant_id, resource=resource).first()
if tenant_quota:
tenant_quota.update({'limit': limit})
else:
tenant_quota = Quota(tenant_id=tenant_id,
resource=resource,
limit=limit)
tenant_quota = quota_models.Quota(tenant_id=tenant_id,
resource=resource,
limit=limit)
context.session.add(tenant_quota)
def _get_quotas(self, context, tenant_id, resources):

0
neutron/tests/unit/db/quota/__init__.py Normal file
View File

229
neutron/tests/unit/db/quota/test_api.py Normal file
View File

@ -0,0 +1,229 @@
# Copyright (c) 2015 OpenStack Foundation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from neutron import context
from neutron.db.quota import api as quota_api
from neutron.tests.unit import testlib_api
class TestQuotaDbApi(testlib_api.SqlTestCaseLight):
def _set_context(self):
self.tenant_id = 'Higuain'
self.context = context.Context('Gonzalo', self.tenant_id,
is_admin=False, is_advsvc=False)
def _create_quota_usage(self, resource, used, reserved, tenant_id=None):
tenant_id = tenant_id or self.tenant_id
return quota_api.set_quota_usage(
self.context, resource, tenant_id,
in_use=used, reserved=reserved)
def _verify_quota_usage(self, usage_info,
expected_resource=None,
expected_used=None,
expected_reserved=None,
expected_dirty=None):
self.assertEqual(self.tenant_id, usage_info.tenant_id)
if expected_resource:
self.assertEqual(expected_resource, usage_info.resource)
if expected_dirty is not None:
self.assertEqual(expected_dirty, usage_info.dirty)
if expected_used is not None:
self.assertEqual(expected_used, usage_info.used)
if expected_reserved is not None:
self.assertEqual(expected_reserved, usage_info.reserved)
if expected_used is not None and expected_reserved is not None:
self.assertEqual(expected_used + expected_reserved,
usage_info.total)
def setUp(self):
super(TestQuotaDbApi, self).setUp()
self._set_context()
def test_create_quota_usage(self):
usage_info = self._create_quota_usage('goals', 26, 10)
self._verify_quota_usage(usage_info,
expected_resource='goals',
expected_used=26,
expected_reserved=10)
def test_update_quota_usage(self):
self._create_quota_usage('goals', 26, 10)
# Higuain scores a double
usage_info_1 = quota_api.set_quota_usage(
self.context, 'goals', self.tenant_id,
in_use=28)
self._verify_quota_usage(usage_info_1,
expected_used=28,
expected_reserved=10)
usage_info_2 = quota_api.set_quota_usage(
self.context, 'goals', self.tenant_id,
reserved=8)
self._verify_quota_usage(usage_info_2,
expected_used=28,
expected_reserved=8)
def test_update_quota_usage_with_deltas(self):
self._create_quota_usage('goals', 26, 10)
# Higuain scores a double
usage_info_1 = quota_api.set_quota_usage(
self.context, 'goals', self.tenant_id,
in_use=2, delta=True)
self._verify_quota_usage(usage_info_1,
expected_used=28,
expected_reserved=10)
usage_info_2 = quota_api.set_quota_usage(
self.context, 'goals', self.tenant_id,
reserved=-2, delta=True)
self._verify_quota_usage(usage_info_2,
expected_used=28,
expected_reserved=8)
def test_set_quota_usage_dirty(self):
self._create_quota_usage('goals', 26, 10)
# Higuain needs a shower after the match
self.assertEqual(1, quota_api.set_quota_usage_dirty(
self.context, 'goals', self.tenant_id))
usage_info = quota_api.get_quota_usage_by_resource_and_tenant(
self.context, 'goals', self.tenant_id)
self._verify_quota_usage(usage_info,
expected_dirty=True)
# Higuain is clean now
self.assertEqual(1, quota_api.set_quota_usage_dirty(
self.context, 'goals', self.tenant_id, dirty=False))
usage_info = quota_api.get_quota_usage_by_resource_and_tenant(
self.context, 'goals', self.tenant_id)
self._verify_quota_usage(usage_info,
expected_dirty=False)
def test_set_dirty_non_existing_quota_usage(self):
self.assertEqual(0, quota_api.set_quota_usage_dirty(
self.context, 'meh', self.tenant_id))
def test_set_resources_quota_usage_dirty(self):
self._create_quota_usage('goals', 26, 10)
self._create_quota_usage('assists', 11, 5)
self._create_quota_usage('bookings', 3, 1)
self.assertEqual(2, quota_api.set_resources_quota_usage_dirty(
self.context, ['goals', 'bookings'], self.tenant_id))
usage_info_goals = quota_api.get_quota_usage_by_resource_and_tenant(
self.context, 'goals', self.tenant_id)
usage_info_assists = quota_api.get_quota_usage_by_resource_and_tenant(
self.context, 'assists', self.tenant_id)
usage_info_bookings = quota_api.get_quota_usage_by_resource_and_tenant(
self.context, 'bookings', self.tenant_id)
self._verify_quota_usage(usage_info_goals, expected_dirty=True)
self._verify_quota_usage(usage_info_assists, expected_dirty=False)
self._verify_quota_usage(usage_info_bookings, expected_dirty=True)
def test_set_resources_quota_usage_dirty_with_empty_list(self):
self._create_quota_usage('goals', 26, 10)
self._create_quota_usage('assists', 11, 5)
self._create_quota_usage('bookings', 3, 1)
# Expect all the resources for the tenant to be set dirty
self.assertEqual(3, quota_api.set_resources_quota_usage_dirty(
self.context, [], self.tenant_id))
usage_info_goals = quota_api.get_quota_usage_by_resource_and_tenant(
self.context, 'goals', self.tenant_id)
usage_info_assists = quota_api.get_quota_usage_by_resource_and_tenant(
self.context, 'assists', self.tenant_id)
usage_info_bookings = quota_api.get_quota_usage_by_resource_and_tenant(
self.context, 'bookings', self.tenant_id)
self._verify_quota_usage(usage_info_goals, expected_dirty=True)
self._verify_quota_usage(usage_info_assists, expected_dirty=True)
self._verify_quota_usage(usage_info_bookings, expected_dirty=True)
# Higuain is clean now
self.assertEqual(1, quota_api.set_quota_usage_dirty(
self.context, 'goals', self.tenant_id, dirty=False))
usage_info = quota_api.get_quota_usage_by_resource_and_tenant(
self.context, 'goals', self.tenant_id)
self._verify_quota_usage(usage_info,
expected_dirty=False)
def _test_set_all_quota_usage_dirty(self, expected):
self._create_quota_usage('goals', 26, 10)
self._create_quota_usage('goals', 12, 6, tenant_id='Callejon')
self.assertEqual(expected, quota_api.set_all_quota_usage_dirty(
self.context, 'goals'))
def test_set_all_quota_usage_dirty(self):
# All goal scorers need a shower after the match, but since this is not
# admin context we can clean only one
self._test_set_all_quota_usage_dirty(expected=1)
def test_get_quota_usage_by_tenant(self):
self._create_quota_usage('goals', 26, 10)
self._create_quota_usage('assists', 11, 5)
# Create a resource for a different tenant
self._create_quota_usage('mehs', 99, 99, tenant_id='buffon')
usage_infos = quota_api.get_quota_usage_by_tenant_id(
self.context, self.tenant_id)
self.assertEqual(2, len(usage_infos))
resources = [info.resource for info in usage_infos]
self.assertIn('goals', resources)
self.assertIn('assists', resources)
def test_get_quota_usage_by_resource(self):
self._create_quota_usage('goals', 26, 10)
self._create_quota_usage('assists', 11, 5)
self._create_quota_usage('goals', 12, 6, tenant_id='Callejon')
usage_infos = quota_api.get_quota_usage_by_resource(
self.context, 'goals')
# Only 1 result expected in tenant context
self.assertEqual(1, len(usage_infos))
self._verify_quota_usage(usage_infos[0],
expected_resource='goals',
expected_used=26,
expected_reserved=10)
def test_get_quota_usage_by_tenant_and_resource(self):
self._create_quota_usage('goals', 26, 10)
usage_info = quota_api.get_quota_usage_by_resource_and_tenant(
self.context, 'goals', self.tenant_id)
self._verify_quota_usage(usage_info,
expected_resource='goals',
expected_used=26,
expected_reserved=10)
def test_get_non_existing_quota_usage_returns_none(self):
self.assertIsNone(quota_api.get_quota_usage_by_resource_and_tenant(
self.context, 'goals', self.tenant_id))
class TestQuotaDbApiAdminContext(TestQuotaDbApi):
def _set_context(self):
self.tenant_id = 'Higuain'
self.context = context.Context('Gonzalo', self.tenant_id,
is_admin=True, is_advsvc=True,
load_admin_roles=False)
def test_get_quota_usage_by_resource(self):
self._create_quota_usage('goals', 26, 10)
self._create_quota_usage('assists', 11, 5)
self._create_quota_usage('goals', 12, 6, tenant_id='Callejon')
usage_infos = quota_api.get_quota_usage_by_resource(
self.context, 'goals')
# 2 results expected in admin context
self.assertEqual(2, len(usage_infos))
for usage_info in usage_infos:
self.assertEqual('goals', usage_info.resource)
def test_set_all_quota_usage_dirty(self):
# All goal scorers need a shower after the match, and with admin
# context we should be able to clean all of them
self._test_set_all_quota_usage_dirty(expected=2)