diff --git a/neutron/api/v2/base.py b/neutron/api/v2/base.py index 69a88d230b2..5f808a2a980 100644 --- a/neutron/api/v2/base.py +++ b/neutron/api/v2/base.py @@ -416,13 +416,15 @@ class Controller(object): if self._collection in body: # Have to account for bulk create items = body[self._collection] - deltas = {} - bulk = True else: items = [body] - bulk = False # Ensure policy engine is initialized policy.init() + # Store requested resource amounts grouping them by tenant + # This won't work with multiple resources. However because of the + # current structure of this controller there will hardly be more than + # one resource for which reservations are being made + request_deltas = {} for item in items: self._validate_network_tenant_ownership(request, item[self._resource]) @@ -433,30 +435,34 @@ class Controller(object): if 'tenant_id' not in item[self._resource]: # no tenant_id - no quota check continue - try: - tenant_id = item[self._resource]['tenant_id'] - count = quota.QUOTAS.count(request.context, self._resource, - self._plugin, tenant_id) - if bulk: - delta = deltas.get(tenant_id, 0) + 1 - deltas[tenant_id] = delta - else: - delta = 1 - kwargs = {self._resource: count + delta} - except exceptions.QuotaResourceUnknown as e: + tenant_id = item[self._resource]['tenant_id'] + delta = request_deltas.get(tenant_id, 0) + delta = delta + 1 + request_deltas[tenant_id] = delta + # Quota enforcement + reservations = [] + try: + for tenant in request_deltas: + reservation = quota.QUOTAS.make_reservation( + request.context, + tenant, + {self._resource: + request_deltas[tenant]}, + self._plugin) + reservations.append(reservation) + except exceptions.QuotaResourceUnknown as e: # We don't want to quota this resource LOG.debug(e) - else: - quota.QUOTAS.limit_check(request.context, - item[self._resource]['tenant_id'], - **kwargs) def notify(create_result): # Ensure usage trackers for all resources affected by this API # operation are marked as dirty - # TODO(salv-orlando): This operation will happen in a single - # transaction with reservation commit once that is implemented - resource_registry.set_resources_dirty(request.context) + with request.context.session.begin(): + # Commit the reservation(s) + for reservation in reservations: + quota.QUOTAS.commit_reservation( + request.context, reservation.reservation_id) + resource_registry.set_resources_dirty(request.context) notifier_method = self._resource + '.create.end' self._notifier.info(request.context, @@ -467,11 +473,35 @@ class Controller(object): notifier_method) return create_result - kwargs = {self._parent_id_name: parent_id} if parent_id else {} + def do_create(body, bulk=False, emulated=False): + kwargs = {self._parent_id_name: parent_id} if parent_id else {} + if bulk and not emulated: + obj_creator = getattr(self._plugin, "%s_bulk" % action) + else: + obj_creator = getattr(self._plugin, action) + try: + if emulated: + return self._emulate_bulk_create(obj_creator, request, + body, parent_id) + else: + if self._collection in body: + # This is weird but fixing it requires changes to the + # plugin interface + kwargs.update({self._collection: body}) + else: + kwargs.update({self._resource: body}) + return obj_creator(request.context, **kwargs) + except Exception: + # In case of failure the plugin will always raise an + # exception. Cancel the reservation + with excutils.save_and_reraise_exception(): + for reservation in reservations: + quota.QUOTAS.cancel_reservation( + request.context, reservation.reservation_id) + if self._collection in body and self._native_bulk: # plugin does atomic bulk create operations - obj_creator = getattr(self._plugin, "%s_bulk" % action) - objs = obj_creator(request.context, body, **kwargs) + objs = do_create(body, bulk=True) # Use first element of list to discriminate attributes which # should be removed because of authZ policies fields_to_strip = self._exclude_attributes_by_policy( @@ -480,15 +510,12 @@ class Controller(object): request.context, obj, fields_to_strip=fields_to_strip) for obj in objs]}) else: - obj_creator = getattr(self._plugin, action) if self._collection in body: # Emulate atomic bulk behavior - objs = self._emulate_bulk_create(obj_creator, request, - body, parent_id) + objs = do_create(body, bulk=True, emulated=True) return notify({self._collection: objs}) else: - kwargs.update({self._resource: body}) - obj = obj_creator(request.context, **kwargs) + obj = do_create(body) self._send_nova_notification(action, {}, {self._resource: obj}) return notify({self._resource: self._view(request.context, diff --git a/neutron/db/migration/alembic_migrations/versions/HEADS b/neutron/db/migration/alembic_migrations/versions/HEADS index aae43e3946f..c4140b06d89 100644 --- a/neutron/db/migration/alembic_migrations/versions/HEADS +++ b/neutron/db/migration/alembic_migrations/versions/HEADS @@ -1,3 +1,3 @@ 2a16083502f3 -48153cb5f051 +9859ac9c136 kilo diff --git a/neutron/db/migration/alembic_migrations/versions/liberty/expand/9859ac9c136_quota_reservations.py b/neutron/db/migration/alembic_migrations/versions/liberty/expand/9859ac9c136_quota_reservations.py new file mode 100644 index 00000000000..c8935a86f13 --- /dev/null +++ b/neutron/db/migration/alembic_migrations/versions/liberty/expand/9859ac9c136_quota_reservations.py @@ -0,0 +1,47 @@ +# Copyright 2015 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""quota_reservations + +Revision ID: 9859ac9c136 +Revises: 48153cb5f051 +Create Date: 2015-03-11 06:40:56.775075 + +""" + +# revision identifiers, used by Alembic. +revision = '9859ac9c136' +down_revision = '48153cb5f051' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table( + 'reservations', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('tenant_id', sa.String(length=255), nullable=True), + sa.Column('expiration', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id')) + + op.create_table( + 'resourcedeltas', + sa.Column('resource', sa.String(length=255), nullable=False), + sa.Column('reservation_id', sa.String(length=36), nullable=False), + sa.Column('amount', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['reservation_id'], ['reservations.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('resource', 'reservation_id')) diff --git a/neutron/db/quota/api.py b/neutron/db/quota/api.py index 40a0a597d38..9657db07959 100644 --- a/neutron/db/quota/api.py +++ b/neutron/db/quota/api.py @@ -13,11 +13,21 @@ # under the License. import collections +import datetime + +import sqlalchemy as sa +from sqlalchemy.orm import exc as orm_exc +from sqlalchemy import sql from neutron.db import common_db_mixin as common_db_api from neutron.db.quota import models as quota_models +# Wrapper for utcnow - needed for mocking it in unit tests +def utcnow(): + return datetime.datetime.utcnow() + + class QuotaUsageInfo(collections.namedtuple( 'QuotaUsageInfo', ['resource', 'tenant_id', 'used', 'reserved', 'dirty'])): @@ -27,6 +37,32 @@ class QuotaUsageInfo(collections.namedtuple( return self.reserved + self.used +class ReservationInfo(object): + """Information about a resource reservation.""" + + def __init__(self, reservation_id, tenant_id, expiration, deltas): + self._reservation_id = reservation_id + self._tenant_id = tenant_id + self._expiration = expiration + self._deltas = deltas + + @property + def reservation_id(self): + return self._reservation_id + + @property + def tenant_id(self): + return self._tenant_id + + @property + def expiration(self): + return self._expiration + + @property + def deltas(self): + return self._deltas + + def get_quota_usage_by_resource_and_tenant(context, resource, tenant_id, lock_for_update=False): """Return usage info for a given resource and tenant. @@ -157,3 +193,105 @@ def set_all_quota_usage_dirty(context, resource, dirty=True): query = common_db_api.model_query(context, quota_models.QuotaUsage) query = query.filter_by(resource=resource) return query.update({'dirty': dirty}) + + +def create_reservation(context, tenant_id, deltas, expiration=None): + # This method is usually called from within another transaction. + # Consider using begin_nested + with context.session.begin(subtransactions=True): + expiration = expiration or (utcnow() + datetime.timedelta(0, 120)) + resv = quota_models.Reservation(tenant_id=tenant_id, + expiration=expiration) + context.session.add(resv) + for (resource, delta) in deltas.items(): + context.session.add( + quota_models.ResourceDelta(resource=resource, + amount=delta, + reservation=resv)) + # quota_usage for all resources involved in this reservation must + # be marked as dirty + set_resources_quota_usage_dirty( + context, deltas.keys(), tenant_id) + return ReservationInfo(resv['id'], + resv['tenant_id'], + resv['expiration'], + dict((delta.resource, delta.amount) + for delta in resv.resource_deltas)) + + +def get_reservation(context, reservation_id): + query = context.session.query(quota_models.Reservation).filter_by( + id=reservation_id) + resv = query.first() + if not resv: + return + return ReservationInfo(resv['id'], + resv['tenant_id'], + resv['expiration'], + dict((delta.resource, delta.amount) + for delta in resv.resource_deltas)) + + +def remove_reservation(context, reservation_id, set_dirty=False): + delete_query = context.session.query(quota_models.Reservation).filter_by( + id=reservation_id) + # Not handling MultipleResultsFound as the query is filtering by primary + # key + try: + reservation = delete_query.one() + except orm_exc.NoResultFound: + # TODO(salv-orlando): Raise here and then handle the exception? + return + tenant_id = reservation.tenant_id + resources = [delta.resource for delta in reservation.resource_deltas] + num_deleted = delete_query.delete() + if set_dirty: + # quota_usage for all resource involved in this reservation must + # be marked as dirty + set_resources_quota_usage_dirty(context, resources, tenant_id) + return num_deleted + + +def get_reservations_for_resources(context, tenant_id, resources, + expired=False): + """Retrieve total amount of reservations for specified resources. + + :param context: Neutron context with db session + :param tenant_id: Tenant identifier + :param resources: Resources for which reserved amounts should be fetched + :param expired: False to fetch active reservations, True to fetch expired + reservations (defaults to False) + :returns: a dictionary mapping resources with corresponding deltas + """ + if not resources: + # Do not waste time + return + now = utcnow() + resv_query = context.session.query( + quota_models.ResourceDelta.resource, + quota_models.Reservation.expiration, + sql.func.sum(quota_models.ResourceDelta.amount)).join( + quota_models.Reservation) + if expired: + exp_expr = (quota_models.Reservation.expiration < now) + else: + exp_expr = (quota_models.Reservation.expiration >= now) + resv_query = resv_query.filter(sa.and_( + quota_models.Reservation.tenant_id == tenant_id, + quota_models.ResourceDelta.resource.in_(resources), + exp_expr)).group_by( + quota_models.ResourceDelta.resource) + return dict((resource, total_reserved) + for (resource, exp, total_reserved) in resv_query) + + +def remove_expired_reservations(context, tenant_id=None): + now = utcnow() + resv_query = context.session.query(quota_models.Reservation) + if tenant_id: + tenant_expr = (quota_models.Reservation.tenant_id == tenant_id) + else: + tenant_expr = sql.true() + resv_query = resv_query.filter(sa.and_( + tenant_expr, quota_models.Reservation.expiration < now)) + return resv_query.delete() diff --git a/neutron/db/quota/driver.py b/neutron/db/quota/driver.py index cf6031ae2d8..a715ba53070 100644 --- a/neutron/db/quota/driver.py +++ b/neutron/db/quota/driver.py @@ -13,9 +13,16 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_db import api as oslo_db_api +from oslo_log import log + from neutron.common import exceptions +from neutron.db import api as db_api +from neutron.db.quota import api as quota_api from neutron.db.quota import models as quota_models +LOG = log.getLogger(__name__) + class DbQuotaDriver(object): """Driver to perform necessary checks to enforce quotas and obtain quota @@ -42,7 +49,8 @@ class DbQuotaDriver(object): # update with tenant specific limits q_qry = context.session.query(quota_models.Quota).filter_by( tenant_id=tenant_id) - tenant_quota.update((q['resource'], q['limit']) for q in q_qry) + for item in q_qry: + tenant_quota[item['resource']] = item['limit'] return tenant_quota @@ -116,6 +124,112 @@ class DbQuotaDriver(object): return dict((k, v) for k, v in quotas.items()) + def _handle_expired_reservations(self, context, tenant_id, + resource, expired_amount): + LOG.debug(("Adjusting usage for resource %(resource)s: " + "removing %(expired)d reserved items"), + {'resource': resource, + 'expired': expired_amount}) + # TODO(salv-orlando): It should be possible to do this + # operation for all resources with a single query. + # Update reservation usage + quota_api.set_quota_usage( + context, + resource, + tenant_id, + reserved=-expired_amount, + delta=True) + # Delete expired reservations (we don't want them to accrue + # in the database) + quota_api.remove_expired_reservations( + context, tenant_id=tenant_id) + + @oslo_db_api.wrap_db_retry(max_retries=db_api.MAX_RETRIES, + retry_on_request=True, + retry_on_deadlock=True) + def make_reservation(self, context, tenant_id, resources, deltas, plugin): + # Lock current reservation table + # NOTE(salv-orlando): This routine uses DB write locks. + # These locks are acquired by the count() method invoked on resources. + # Please put your shotguns aside. + # A non locking algorithm for handling reservation is feasible, however + # it will require two database writes even in cases when there are not + # concurrent reservations. + # For this reason it might be advisable to handle contention using + # this kind of locks and paying the cost of a write set certification + # failure when a mysql galera cluster is employed. Also, this class of + # locks should be ok to use when support for sending "hotspot" writes + # to a single node will be avaialable. + requested_resources = deltas.keys() + with context.session.begin(): + # Gather current usage information + # TODO(salv-orlando): calling count() for every resource triggers + # multiple queries on quota usage. This should be improved, however + # this is not an urgent matter as the REST API currently only + # allows allocation of a resource at a time + # NOTE: pass plugin too for compatibility with CountableResource + # instances + current_usages = dict( + (resource, resources[resource].count( + context, plugin, tenant_id)) for + resource in requested_resources) + # get_tenant_quotes needs in inout a dictionary mapping resource + # name to BaseResosurce instances so that the default quota can be + # retrieved + current_limits = self.get_tenant_quotas( + context, resources, tenant_id) + # Adjust for expired reservations. Apparently it is cheaper than + # querying everytime for active reservations and counting overall + # quantity of resources reserved + expired_deltas = quota_api.get_reservations_for_resources( + context, tenant_id, requested_resources, expired=True) + # Verify that the request can be accepted with current limits + resources_over_limit = [] + for resource in requested_resources: + expired_reservations = expired_deltas.get(resource, 0) + total_usage = current_usages[resource] - expired_reservations + # A negative quota limit means infinite + if current_limits[resource] < 0: + LOG.debug(("Resource %(resource)s has unlimited quota " + "limit. It is possible to allocate %(delta)s " + "items."), {'resource': resource, + 'delta': deltas[resource]}) + continue + res_headroom = current_limits[resource] - total_usage + LOG.debug(("Attempting to reserve %(delta)d items for " + "resource %(resource)s. Total usage: %(total)d; " + "quota limit: %(limit)d; headroom:%(headroom)d"), + {'resource': resource, + 'delta': deltas[resource], + 'total': total_usage, + 'limit': current_limits[resource], + 'headroom': res_headroom}) + if res_headroom < deltas[resource]: + resources_over_limit.append(resource) + if expired_reservations: + self._handle_expired_reservations( + context, tenant_id, resource, expired_reservations) + + if resources_over_limit: + raise exceptions.OverQuota(overs=sorted(resources_over_limit)) + # Success, store the reservation + # TODO(salv-orlando): Make expiration time configurable + return quota_api.create_reservation( + context, tenant_id, deltas) + + def commit_reservation(self, context, reservation_id): + # Do not mark resource usage as dirty. If a reservation is committed, + # then the releveant resources have been created. Usage data for these + # resources has therefore already been marked dirty. + quota_api.remove_reservation(context, reservation_id, + set_dirty=False) + + def cancel_reservation(self, context, reservation_id): + # Mark resource usage as dirty so the next time both actual resources + # used and reserved will be recalculated + quota_api.remove_reservation(context, reservation_id, + set_dirty=True) + def limit_check(self, context, tenant_id, resources, values): """Check simple quota limits. diff --git a/neutron/db/quota/models.py b/neutron/db/quota/models.py index b0abd0d9f54..a4dbd7117e4 100644 --- a/neutron/db/quota/models.py +++ b/neutron/db/quota/models.py @@ -13,12 +13,33 @@ # under the License. import sqlalchemy as sa +from sqlalchemy import orm from sqlalchemy import sql from neutron.db import model_base from neutron.db import models_v2 +class ResourceDelta(model_base.BASEV2): + resource = sa.Column(sa.String(255), primary_key=True) + reservation_id = sa.Column(sa.String(36), + sa.ForeignKey('reservations.id', + ondelete='CASCADE'), + primary_key=True, + nullable=False) + # Requested amount of resource + amount = sa.Column(sa.Integer) + + +class Reservation(model_base.BASEV2, models_v2.HasId): + tenant_id = sa.Column(sa.String(255)) + expiration = sa.Column(sa.DateTime()) + resource_deltas = orm.relationship(ResourceDelta, + backref='reservation', + lazy="joined", + cascade='all, delete-orphan') + + class Quota(model_base.BASEV2, models_v2.HasId, models_v2.HasTenant): """Represent a single quota override for a tenant. diff --git a/neutron/quota/__init__.py b/neutron/quota/__init__.py index 97b466e872a..df54d9f9128 100644 --- a/neutron/quota/__init__.py +++ b/neutron/quota/__init__.py @@ -24,6 +24,7 @@ import six import webob from neutron.common import exceptions +from neutron.db.quota import api as quota_api from neutron.i18n import _LI, _LW from neutron.quota import resource_registry @@ -152,6 +153,33 @@ class ConfDriver(object): msg = _('Access to this resource was denied.') raise webob.exc.HTTPForbidden(msg) + def make_reservation(self, context, tenant_id, resources, deltas, plugin): + """This driver does not support reservations. + + This routine is provided for backward compatibility purposes with + the API controllers which have now been adapted to make reservations + rather than counting resources and checking limits - as this + routine ultimately does. + """ + for resource in deltas.keys(): + count = QUOTAS.count(context, resource, plugin, tenant_id) + total_use = deltas.get(resource, 0) + count + deltas[resource] = total_use + + self.limit_check( + context, + tenant_id, + resource_registry.get_all_resources(), + deltas) + # return a fake reservation - the REST controller expects it + return quota_api.ReservationInfo('fake', None, None, None) + + def commit_reservation(self, context, reservation_id): + """Tnis is a noop as this driver does not support reservations.""" + + def cancel_reservation(self, context, reservation_id): + """Tnis is a noop as this driver does not support reservations.""" + class QuotaEngine(object): """Represent the set of recognized quotas.""" @@ -210,6 +238,39 @@ class QuotaEngine(object): return res.count(context, *args, **kwargs) + def make_reservation(self, context, tenant_id, deltas, plugin): + # Verify that resources are managed by the quota engine + # Ensure no value is less than zero + unders = [key for key, val in deltas.items() if val < 0] + if unders: + raise exceptions.InvalidQuotaValue(unders=sorted(unders)) + + requested_resources = set(deltas.keys()) + all_resources = resource_registry.get_all_resources() + managed_resources = set([res for res in all_resources.keys() + if res in requested_resources]) + # Make sure we accounted for all of them... + unknown_resources = requested_resources - managed_resources + + if unknown_resources: + raise exceptions.QuotaResourceUnknown( + unknown=sorted(unknown_resources)) + # FIXME(salv-orlando): There should be no reason for sending all the + # resource in the registry to the quota driver, but as other driver + # APIs request them, this will be sorted out with a different patch. + return self.get_driver().make_reservation( + context, + tenant_id, + all_resources, + deltas, + plugin) + + def commit_reservation(self, context, reservation_id): + self.get_driver().commit_reservation(context, reservation_id) + + def cancel_reservation(self, context, reservation_id): + self.get_driver().cancel_reservation(context, reservation_id) + def limit_check(self, context, tenant_id, **values): """Check simple quota limits. @@ -232,6 +293,7 @@ class QuotaEngine(object): :param tenant_id: Tenant for which the quota limit is being checked :param values: Dict specifying requested deltas for each resource """ + # TODO(salv-orlando): Deprecate calls to this API # Verify that resources are managed by the quota engine requested_resources = set(values.keys()) managed_resources = set([res for res in diff --git a/neutron/quota/resource.py b/neutron/quota/resource.py index eb0036859fa..0030307ba69 100644 --- a/neutron/quota/resource.py +++ b/neutron/quota/resource.py @@ -208,14 +208,15 @@ class TrackedResource(BaseResource): max_retries=db_api.MAX_RETRIES, exception_checker=lambda exc: isinstance(exc, oslo_db_exception.DBDuplicateEntry)) - def _set_quota_usage(self, context, tenant_id, in_use): - return quota_api.set_quota_usage(context, self.name, - tenant_id, in_use=in_use) + def _set_quota_usage(self, context, tenant_id, in_use, reserved): + return quota_api.set_quota_usage(context, self.name, tenant_id, + in_use=in_use, reserved=reserved) - def _resync(self, context, tenant_id, in_use): + def _resync(self, context, tenant_id, in_use, reserved): # Update quota usage usage_info = self._set_quota_usage( - context, tenant_id, in_use=in_use) + context, tenant_id, in_use, reserved) + self._dirty_tenants.discard(tenant_id) self._out_of_sync_tenants.discard(tenant_id) LOG.debug(("Unset dirty status for tenant:%(tenant_id)s on " @@ -231,40 +232,62 @@ class TrackedResource(BaseResource): {'tenant_id': tenant_id, 'resource': self.name}) in_use = context.session.query(self._model_class).filter_by( tenant_id=tenant_id).count() + reservations = quota_api.get_reservations_for_resources( + context, tenant_id, [self.name]) + reserved = reservations.get(self.name, 0) # Update quota usage - return self._resync(context, tenant_id, in_use) + return self._resync(context, tenant_id, in_use, reserved) def count(self, context, _plugin, tenant_id, resync_usage=False): - """Return the current usage count for the resource.""" - # Load current usage data + """Return the current usage count for the resource. + + This method will fetch the information from resource usage data, + unless usage data are marked as "dirty", in which case both used and + reserved resource are explicitly counted. + + The _plugin and _resource parameters are unused but kept for + compatibility with the signature of the count method for + CountableResource instances. + """ + # Load current usage data, setting a row-level lock on the DB usage_info = quota_api.get_quota_usage_by_resource_and_tenant( - context, self.name, tenant_id) + context, self.name, tenant_id, lock_for_update=True) # If dirty or missing, calculate actual resource usage querying # the database and set/create usage info data # NOTE: this routine "trusts" usage counters at service startup. This # assumption is generally valid, but if the database is tampered with, # or if data migrations do not take care of usage counters, the # assumption will not hold anymore - if (tenant_id in self._dirty_tenants or not usage_info - or usage_info.dirty): + if (tenant_id in self._dirty_tenants or + not usage_info or usage_info.dirty): LOG.debug(("Usage tracker for resource:%(resource)s and tenant:" "%(tenant_id)s is out of sync, need to count used " "quota"), {'resource': self.name, 'tenant_id': tenant_id}) in_use = context.session.query(self._model_class).filter_by( tenant_id=tenant_id).count() + reservations = quota_api.get_reservations_for_resources( + context, tenant_id, [self.name]) + reserved = reservations.get(self.name, 0) + # Update quota usage, if requested (by default do not do that, as # typically one counts before adding a record, and that would mark # the usage counter as dirty again) if resync_usage or not usage_info: - usage_info = self._resync(context, tenant_id, in_use) + usage_info = self._resync(context, tenant_id, + in_use, reserved) else: usage_info = quota_api.QuotaUsageInfo(usage_info.resource, usage_info.tenant_id, in_use, - usage_info.reserved, + reserved, usage_info.dirty) + LOG.debug(("Quota usage for %(resource)s was recalculated. " + "Used quota:%(used)d; Reserved quota:%(reserved)d"), + {'resource': self.name, + 'used': usage_info.used, + 'reserved': usage_info.reserved}) return usage_info.total def register_events(self): diff --git a/neutron/quota/resource_registry.py b/neutron/quota/resource_registry.py index d0263e87614..6154749ba95 100644 --- a/neutron/quota/resource_registry.py +++ b/neutron/quota/resource_registry.py @@ -65,7 +65,7 @@ def set_resources_dirty(context): return for res in get_all_resources().values(): - with context.session.begin(): + with context.session.begin(subtransactions=True): if is_tracked(res.name) and res.dirty: res.mark_dirty(context, nested=True) diff --git a/neutron/tests/unit/db/quota/test_api.py b/neutron/tests/unit/db/quota/test_api.py index a64e2b98b44..c527a663179 100644 --- a/neutron/tests/unit/db/quota/test_api.py +++ b/neutron/tests/unit/db/quota/test_api.py @@ -12,6 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. +import datetime + +import mock + from neutron import context from neutron.db.quota import api as quota_api from neutron.tests.unit import testlib_api @@ -24,6 +28,12 @@ class TestQuotaDbApi(testlib_api.SqlTestCaseLight): self.context = context.Context('Gonzalo', self.tenant_id, is_admin=False, is_advsvc=False) + def _create_reservation(self, resource_deltas, + tenant_id=None, expiration=None): + tenant_id = tenant_id or self.tenant_id + return quota_api.create_reservation( + self.context, tenant_id, resource_deltas, expiration) + def _create_quota_usage(self, resource, used, reserved, tenant_id=None): tenant_id = tenant_id or self.tenant_id return quota_api.set_quota_usage( @@ -203,6 +213,125 @@ class TestQuotaDbApi(testlib_api.SqlTestCaseLight): self.assertIsNone(quota_api.get_quota_usage_by_resource_and_tenant( self.context, 'goals', self.tenant_id)) + def _verify_reserved_resources(self, expected, actual): + for (resource, delta) in actual.items(): + self.assertIn(resource, expected) + self.assertEqual(delta, expected[resource]) + del expected[resource] + self.assertFalse(expected) + + def test_create_reservation(self): + resources = {'goals': 2, 'assists': 1} + resv = self._create_reservation(resources) + self.assertEqual(self.tenant_id, resv.tenant_id) + self._verify_reserved_resources(resources, resv.deltas) + + def test_create_reservation_with_expirtion(self): + resources = {'goals': 2, 'assists': 1} + exp_date = datetime.datetime(2016, 3, 31, 14, 30) + resv = self._create_reservation(resources, expiration=exp_date) + self.assertEqual(self.tenant_id, resv.tenant_id) + self.assertEqual(exp_date, resv.expiration) + self._verify_reserved_resources(resources, resv.deltas) + + def _test_remove_reservation(self, set_dirty): + resources = {'goals': 2, 'assists': 1} + resv = self._create_reservation(resources) + self.assertEqual(1, quota_api.remove_reservation( + self.context, resv.reservation_id, set_dirty=set_dirty)) + + def test_remove_reservation(self): + self._test_remove_reservation(False) + + def test_remove_reservation_and_set_dirty(self): + routine = 'neutron.db.quota.api.set_resources_quota_usage_dirty' + with mock.patch(routine) as mock_routine: + self._test_remove_reservation(False) + mock_routine.assert_called_once_with( + self.context, mock.ANY, self.tenant_id) + + def test_remove_non_existent_reservation(self): + self.assertIsNone(quota_api.remove_reservation(self.context, 'meh')) + + def _get_reservations_for_resource_helper(self): + # create three reservation, 1 expired + resources_1 = {'goals': 2, 'assists': 1} + resources_2 = {'goals': 3, 'bookings': 1} + resources_3 = {'bookings': 2, 'assists': 2} + exp_date_1 = datetime.datetime(2016, 3, 31, 14, 30) + exp_date_2 = datetime.datetime(2015, 3, 31, 14, 30) + self._create_reservation(resources_1, expiration=exp_date_1) + self._create_reservation(resources_2, expiration=exp_date_1) + self._create_reservation(resources_3, expiration=exp_date_2) + + def test_get_reservations_for_resources(self): + with mock.patch('neutron.db.quota.api.utcnow') as mock_utcnow: + self._get_reservations_for_resource_helper() + mock_utcnow.return_value = datetime.datetime( + 2015, 5, 20, 0, 0) + deltas = quota_api.get_reservations_for_resources( + self.context, self.tenant_id, ['goals', 'assists', 'bookings']) + self.assertIn('goals', deltas) + self.assertEqual(5, deltas['goals']) + self.assertIn('assists', deltas) + self.assertEqual(1, deltas['assists']) + self.assertIn('bookings', deltas) + self.assertEqual(1, deltas['bookings']) + self.assertEqual(3, len(deltas)) + + def test_get_expired_reservations_for_resources(self): + with mock.patch('neutron.db.quota.api.utcnow') as mock_utcnow: + mock_utcnow.return_value = datetime.datetime( + 2015, 5, 20, 0, 0) + self._get_reservations_for_resource_helper() + deltas = quota_api.get_reservations_for_resources( + self.context, self.tenant_id, + ['goals', 'assists', 'bookings'], + expired=True) + self.assertIn('assists', deltas) + self.assertEqual(2, deltas['assists']) + self.assertIn('bookings', deltas) + self.assertEqual(2, deltas['bookings']) + self.assertEqual(2, len(deltas)) + + def test_get_reservation_for_resources_with_empty_list(self): + self.assertIsNone(quota_api.get_reservations_for_resources( + self.context, self.tenant_id, [])) + + def test_remove_expired_reservations(self): + with mock.patch('neutron.db.quota.api.utcnow') as mock_utcnow: + mock_utcnow.return_value = datetime.datetime( + 2015, 5, 20, 0, 0) + resources = {'goals': 2, 'assists': 1} + exp_date_1 = datetime.datetime(2016, 3, 31, 14, 30) + resv_1 = self._create_reservation(resources, expiration=exp_date_1) + exp_date_2 = datetime.datetime(2015, 3, 31, 14, 30) + resv_2 = self._create_reservation(resources, expiration=exp_date_2) + self.assertEqual(1, quota_api.remove_expired_reservations( + self.context, self.tenant_id)) + self.assertIsNone(quota_api.get_reservation( + self.context, resv_2.reservation_id)) + self.assertIsNotNone(quota_api.get_reservation( + self.context, resv_1.reservation_id)) + + def test_remove_expired_reservations_no_tenant(self): + with mock.patch('neutron.db.quota.api.utcnow') as mock_utcnow: + mock_utcnow.return_value = datetime.datetime( + 2015, 5, 20, 0, 0) + resources = {'goals': 2, 'assists': 1} + exp_date_1 = datetime.datetime(2014, 3, 31, 14, 30) + resv_1 = self._create_reservation(resources, expiration=exp_date_1) + exp_date_2 = datetime.datetime(2015, 3, 31, 14, 30) + resv_2 = self._create_reservation(resources, + expiration=exp_date_2, + tenant_id='Callejon') + self.assertEqual(2, quota_api.remove_expired_reservations( + self.context)) + self.assertIsNone(quota_api.get_reservation( + self.context, resv_2.reservation_id)) + self.assertIsNone(quota_api.get_reservation( + self.context, resv_1.reservation_id)) + class TestQuotaDbApiAdminContext(TestQuotaDbApi): diff --git a/neutron/tests/unit/db/quota/test_driver.py b/neutron/tests/unit/db/quota/test_driver.py index 31a741721ce..dafee362a6d 100644 --- a/neutron/tests/unit/db/quota/test_driver.py +++ b/neutron/tests/unit/db/quota/test_driver.py @@ -27,16 +27,22 @@ class FakePlugin(base_plugin.NeutronDbPluginV2, driver.DbQuotaDriver): class TestResource(object): """Describe a test resource for quota checking.""" - def __init__(self, name, default): + def __init__(self, name, default, fake_count=0): self.name = name self.quota = default + self.fake_count = fake_count @property def default(self): return self.quota + def count(self, *args, **kwargs): + return self.fake_count + + PROJECT = 'prj_test' RESOURCE = 'res_test' +ALT_RESOURCE = 'res_test_meh' class TestDbQuotaDriver(testlib_api.SqlTestCase): @@ -132,3 +138,63 @@ class TestDbQuotaDriver(testlib_api.SqlTestCase): self.assertRaises(exceptions.InvalidQuotaValue, self.plugin.limit_check, context.get_admin_context(), PROJECT, resources, values) + + def _test_make_reservation_success(self, quota_driver, + resource_name, deltas): + resources = {resource_name: TestResource(resource_name, 2)} + self.plugin.update_quota_limit(self.context, PROJECT, resource_name, 2) + reservation = quota_driver.make_reservation( + self.context, + self.context.tenant_id, + resources, + deltas, + self.plugin) + self.assertIn(resource_name, reservation.deltas) + self.assertEqual(deltas[resource_name], + reservation.deltas[resource_name]) + self.assertEqual(self.context.tenant_id, + reservation.tenant_id) + + def test_make_reservation_single_resource(self): + quota_driver = driver.DbQuotaDriver() + self._test_make_reservation_success( + quota_driver, RESOURCE, {RESOURCE: 1}) + + def test_make_reservation_fill_quota(self): + quota_driver = driver.DbQuotaDriver() + self._test_make_reservation_success( + quota_driver, RESOURCE, {RESOURCE: 2}) + + def test_make_reservation_multiple_resources(self): + quota_driver = driver.DbQuotaDriver() + resources = {RESOURCE: TestResource(RESOURCE, 2), + ALT_RESOURCE: TestResource(ALT_RESOURCE, 2)} + deltas = {RESOURCE: 1, ALT_RESOURCE: 2} + self.plugin.update_quota_limit(self.context, PROJECT, RESOURCE, 2) + self.plugin.update_quota_limit(self.context, PROJECT, ALT_RESOURCE, 2) + reservation = quota_driver.make_reservation( + self.context, + self.context.tenant_id, + resources, + deltas, + self.plugin) + self.assertIn(RESOURCE, reservation.deltas) + self.assertIn(ALT_RESOURCE, reservation.deltas) + self.assertEqual(1, reservation.deltas[RESOURCE]) + self.assertEqual(2, reservation.deltas[ALT_RESOURCE]) + self.assertEqual(self.context.tenant_id, + reservation.tenant_id) + + def test_make_reservation_over_quota_fails(self): + quota_driver = driver.DbQuotaDriver() + resources = {RESOURCE: TestResource(RESOURCE, 2, + fake_count=2)} + deltas = {RESOURCE: 1} + self.plugin.update_quota_limit(self.context, PROJECT, RESOURCE, 2) + self.assertRaises(exceptions.OverQuota, + quota_driver.make_reservation, + self.context, + self.context.tenant_id, + resources, + deltas, + self.plugin) diff --git a/neutron/tests/unit/extensions/test_quotasv2.py b/neutron/tests/unit/extensions/test_quotasv2.py index e0780e1ee78..8e0e55b3462 100644 --- a/neutron/tests/unit/extensions/test_quotasv2.py +++ b/neutron/tests/unit/extensions/test_quotasv2.py @@ -344,6 +344,24 @@ class QuotaExtensionDbTestCase(QuotaExtensionTestCase): extra_environ=env, expect_errors=True) self.assertEqual(400, res.status_int) + def test_make_reservation_resource_unknown_raises(self): + tenant_id = 'tenant_id1' + self.assertRaises(exceptions.QuotaResourceUnknown, + quota.QUOTAS.make_reservation, + context.get_admin_context(load_admin_roles=False), + tenant_id, + {'foobar': 1}, + plugin=None) + + def test_make_reservation_negative_delta_raises(self): + tenant_id = 'tenant_id1' + self.assertRaises(exceptions.InvalidQuotaValue, + quota.QUOTAS.make_reservation, + context.get_admin_context(load_admin_roles=False), + tenant_id, + {'network': -1}, + plugin=None) + class QuotaExtensionCfgTestCase(QuotaExtensionTestCase): fmt = 'json' diff --git a/neutron/tests/unit/quota/test_resource.py b/neutron/tests/unit/quota/test_resource.py index 7f668539807..88a00bbc924 100644 --- a/neutron/tests/unit/quota/test_resource.py +++ b/neutron/tests/unit/quota/test_resource.py @@ -165,7 +165,8 @@ class TestTrackedResource(testlib_api.SqlTestCaseLight): res.count(self.context, None, self.tenant_id, resync_usage=True) mock_set_quota_usage.assert_called_once_with( - self.context, self.resource, self.tenant_id, in_use=2) + self.context, self.resource, self.tenant_id, + reserved=0, in_use=2) def test_count_with_dirty_true_no_usage_info(self): res = self._create_resource() @@ -184,7 +185,8 @@ class TestTrackedResource(testlib_api.SqlTestCaseLight): self.tenant_id) res.count(self.context, None, self.tenant_id, resync_usage=True) mock_set_quota_usage.assert_called_once_with( - self.context, self.resource, self.tenant_id, in_use=2) + self.context, self.resource, self.tenant_id, + reserved=0, in_use=2) def test_add_delete_data_triggers_event(self): res = self._create_resource() @@ -251,4 +253,5 @@ class TestTrackedResource(testlib_api.SqlTestCaseLight): # and now it should be in sync self.assertNotIn(self.tenant_id, res._out_of_sync_tenants) mock_set_quota_usage.assert_called_once_with( - self.context, self.resource, self.tenant_id, in_use=2) + self.context, self.resource, self.tenant_id, + reserved=0, in_use=2)