From 3c584084c3c79489a7c81b323df5a15b44011e9f Mon Sep 17 00:00:00 2001 From: Salvatore Orlando Date: Thu, 16 Apr 2015 11:56:27 -0700 Subject: [PATCH] Introduce usage data tracking for Neutron This patch introduces application logic support for tracking Neutron resource usage data, thus introducing a different way of enforcing quota limits, which now relies on records tracking resource usage for each tenant. When these records go "out-of-sync" with actual resource usage, the quota usage table entry is marked "dirty". And it will be resynchronized the next time a resource count is requested. Changes in resource utilization are detected using SQLAlchemy events. This patch however does not automatically enable resource usage tracking for any plugin. Plugins must explicitly declare for which resources they wish to enable tracking. To this aim, this patch provides the @tracked_resources decorator. As some operators might wish to not use resource usage tracking, this patch adds the track_quota_usage configuration option. If set to False, DB-level usage tracking will not be performed, and resources will be counted as before, ie: invoking a plugin method. Usage tracking is performed using a new resource type, TrackedResource, which can work side by side with CountableResource, the one Neutron used so far. To this aim a ResourceRegistry class has been introduced for registering and managing resources. Most of the resource management code previously embedded in the QuotaEngine class is being moved to ResourceRegistry as a part of this patch. Partially implements blueprint better-quota Change-Id: If461399900973a38d7b82e0b3ac54fc052f09529 --- neutron/api/rpc/handlers/dhcp_rpc.py | 3 +- neutron/api/v2/base.py | 29 ++- neutron/api/v2/resource_helper.py | 4 +- neutron/api/v2/router.py | 4 +- neutron/db/ipam_backend_mixin.py | 2 +- neutron/db/models_v2.py | 3 +- neutron/extensions/quotasv2.py | 10 +- neutron/extensions/securitygroup.py | 4 +- .../plugins/nec/extensions/packetfilter.py | 11 +- neutron/quota/__init__.py | 137 ++-------- neutron/quota/resource.py | 241 +++++++++++++++++ neutron/quota/resource_registry.py | 245 ++++++++++++++++++ neutron/tests/unit/api/v2/test_base.py | 27 +- .../unit/extensions/extensionattribute.py | 4 +- .../tests/unit/extensions/test_quotasv2.py | 3 +- neutron/tests/unit/quota/__init__.py | 28 ++ neutron/tests/unit/quota/test_resource.py | 232 +++++++++++++++++ .../unit/quota/test_resource_registry.py | 159 ++++++++++++ 18 files changed, 1013 insertions(+), 133 deletions(-) create mode 100644 neutron/quota/resource.py create mode 100644 neutron/quota/resource_registry.py create mode 100644 neutron/tests/unit/quota/__init__.py create mode 100644 neutron/tests/unit/quota/test_resource.py create mode 100644 neutron/tests/unit/quota/test_resource_registry.py diff --git a/neutron/api/rpc/handlers/dhcp_rpc.py b/neutron/api/rpc/handlers/dhcp_rpc.py index 7d97b7c5226..07438334a38 100644 --- a/neutron/api/rpc/handlers/dhcp_rpc.py +++ b/neutron/api/rpc/handlers/dhcp_rpc.py @@ -29,7 +29,7 @@ from neutron.common import utils from neutron.extensions import portbindings from neutron.i18n import _LW from neutron import manager - +from neutron.quota import resource_registry LOG = logging.getLogger(__name__) @@ -203,6 +203,7 @@ class DhcpRpcCallback(object): LOG.warning(_LW('Updating lease expiration is now deprecated. Issued ' 'from host %s.'), host) + @resource_registry.mark_resources_dirty def create_dhcp_port(self, context, **kwargs): """Create and return dhcp port information. diff --git a/neutron/api/v2/base.py b/neutron/api/v2/base.py index 3f72e08f7b0..a1841b8cb8d 100644 --- a/neutron/api/v2/base.py +++ b/neutron/api/v2/base.py @@ -34,6 +34,7 @@ from neutron.db import api as db_api from neutron.i18n import _LE, _LI from neutron import policy from neutron import quota +from neutron.quota import resource_registry LOG = logging.getLogger(__name__) @@ -207,7 +208,15 @@ class Controller(object): name, resource, pluralized=self._collection) - return getattr(self._plugin, name)(*arg_list, **kwargs) + ret_value = getattr(self._plugin, name)(*arg_list, **kwargs) + # It is simply impossible to predict whether one of this + # actions alters resource usage. For instance a tenant port + # is created when a router interface is added. Therefore it is + # important to mark as dirty resources whose counters have + # been altered by this operation + resource_registry.set_resources_dirty(request.context) + return ret_value + return _handle_action else: raise AttributeError() @@ -280,6 +289,9 @@ class Controller(object): pagination_links = pagination_helper.get_links(obj_list) if pagination_links: collection[self._collection + "_links"] = pagination_links + # Synchronize usage trackers, if needed + resource_registry.resync_resource( + request.context, self._resource, request.context.tenant_id) return collection def _item(self, request, id, do_authz=False, field_list=None, @@ -436,6 +448,12 @@ class Controller(object): **kwargs) def notify(create_result): + # Ensure usage trackers for all resources affected by this API + # operation are marked as dirty + # TODO(salv-orlando): This operation will happen in a single + # transaction with reservation commit once that is implemented + resource_registry.set_resources_dirty(request.context) + notifier_method = self._resource + '.create.end' self._notifier.info(request.context, notifier_method, @@ -497,6 +515,9 @@ class Controller(object): obj_deleter = getattr(self._plugin, action) obj_deleter(request.context, id, **kwargs) + # A delete operation usually alters resource usage, so mark affected + # usage trackers as dirty + resource_registry.set_resources_dirty(request.context) notifier_method = self._resource + '.delete.end' self._notifier.info(request.context, notifier_method, @@ -561,6 +582,12 @@ class Controller(object): if parent_id: kwargs[self._parent_id_name] = parent_id obj = obj_updater(request.context, id, **kwargs) + # Usually an update operation does not alter resource usage, but as + # there might be side effects it might be worth checking for changes + # in resource usage here as well (e.g: a tenant port is created when a + # router interface is added) + resource_registry.set_resources_dirty(request.context) + result = {self._resource: self._view(request.context, obj)} notifier_method = self._resource + '.update.end' self._notifier.info(request.context, notifier_method, result) diff --git a/neutron/api/v2/resource_helper.py b/neutron/api/v2/resource_helper.py index 05e403d030d..c506320c91d 100644 --- a/neutron/api/v2/resource_helper.py +++ b/neutron/api/v2/resource_helper.py @@ -20,7 +20,7 @@ from neutron.api import extensions from neutron.api.v2 import base from neutron import manager from neutron.plugins.common import constants -from neutron import quota +from neutron.quota import resource_registry LOG = logging.getLogger(__name__) @@ -80,7 +80,7 @@ def build_resource_info(plural_mappings, resource_map, which_service, if translate_name: collection_name = collection_name.replace('_', '-') if register_quota: - quota.QUOTAS.register_resource_by_name(resource_name) + resource_registry.register_resource_by_name(resource_name) member_actions = action_map.get(resource_name, {}) controller = base.create_resource( collection_name, resource_name, plugin, params, diff --git a/neutron/api/v2/router.py b/neutron/api/v2/router.py index c76f2d02ac5..bd59d854b0e 100644 --- a/neutron/api/v2/router.py +++ b/neutron/api/v2/router.py @@ -27,7 +27,7 @@ from neutron.api.v2 import attributes from neutron.api.v2 import base from neutron import manager from neutron import policy -from neutron import quota +from neutron.quota import resource_registry from neutron import wsgi @@ -106,7 +106,7 @@ class APIRouter(wsgi.Router): _map_resource(RESOURCES[resource], resource, attributes.RESOURCE_ATTRIBUTE_MAP.get( RESOURCES[resource], dict())) - quota.QUOTAS.register_resource_by_name(resource) + resource_registry.register_resource_by_name(resource) for resource in SUB_RESOURCES: _map_resource(SUB_RESOURCES[resource]['collection_name'], resource, diff --git a/neutron/db/ipam_backend_mixin.py b/neutron/db/ipam_backend_mixin.py index ca69f460b78..4d732ec1bec 100644 --- a/neutron/db/ipam_backend_mixin.py +++ b/neutron/db/ipam_backend_mixin.py @@ -409,7 +409,7 @@ class IpamBackendMixin(db_base_plugin_common.DbBasePluginCommon): enable_eagerloads(False).filter_by(id=port_id)) if not context.is_admin: query = query.filter_by(tenant_id=context.tenant_id) - query.delete() + context.session.delete(query.first()) def _save_subnet(self, context, network, diff --git a/neutron/db/models_v2.py b/neutron/db/models_v2.py index 8ba70db7790..23bcde49117 100644 --- a/neutron/db/models_v2.py +++ b/neutron/db/models_v2.py @@ -133,7 +133,8 @@ class Port(model_base.BASEV2, HasId, HasTenant): name = sa.Column(sa.String(attr.NAME_MAX_LEN)) network_id = sa.Column(sa.String(36), sa.ForeignKey("networks.id"), nullable=False) - fixed_ips = orm.relationship(IPAllocation, backref='port', lazy='joined') + fixed_ips = orm.relationship(IPAllocation, backref='port', lazy='joined', + passive_deletes='all') mac_address = sa.Column(sa.String(32), nullable=False) admin_state_up = sa.Column(sa.Boolean(), nullable=False) status = sa.Column(sa.String(16), nullable=False) diff --git a/neutron/extensions/quotasv2.py b/neutron/extensions/quotasv2.py index a47a1adb98c..f9a3ae9915f 100644 --- a/neutron/extensions/quotasv2.py +++ b/neutron/extensions/quotasv2.py @@ -25,6 +25,7 @@ from neutron.common import constants as const from neutron.common import exceptions as n_exc from neutron import manager from neutron import quota +from neutron.quota import resource_registry from neutron import wsgi @@ -48,7 +49,7 @@ class QuotaSetsController(wsgi.Controller): self._update_extended_attributes = True def _update_attributes(self): - for quota_resource in QUOTAS.resources.keys(): + for quota_resource in resource_registry.get_all_resources().keys(): attr_dict = EXTENDED_ATTRIBUTES_2_0[RESOURCE_COLLECTION] attr_dict[quota_resource] = { 'allow_post': False, @@ -60,7 +61,9 @@ class QuotaSetsController(wsgi.Controller): def _get_quotas(self, request, tenant_id): return self._driver.get_tenant_quotas( - request.context, QUOTAS.resources, tenant_id) + request.context, + resource_registry.get_all_resources(), + tenant_id) def create(self, request, body=None): msg = _('POST requests are not supported on this resource.') @@ -70,7 +73,8 @@ class QuotaSetsController(wsgi.Controller): context = request.context self._check_admin(context) return {self._resource_name + "s": - self._driver.get_all_quotas(context, QUOTAS.resources)} + self._driver.get_all_quotas( + context, resource_registry.get_all_resources())} def tenant(self, request): """Retrieve the tenant info in context.""" diff --git a/neutron/extensions/securitygroup.py b/neutron/extensions/securitygroup.py index 8e863e831d6..f199f12025a 100644 --- a/neutron/extensions/securitygroup.py +++ b/neutron/extensions/securitygroup.py @@ -26,7 +26,7 @@ from neutron.api.v2 import base from neutron.common import constants as const from neutron.common import exceptions as nexception from neutron import manager -from neutron import quota +from neutron.quota import resource_registry # Security group Exceptions @@ -305,7 +305,7 @@ class Securitygroup(extensions.ExtensionDescriptor): for resource_name in ['security_group', 'security_group_rule']: collection_name = resource_name.replace('_', '-') + "s" params = RESOURCE_ATTRIBUTE_MAP.get(resource_name + "s", dict()) - quota.QUOTAS.register_resource_by_name(resource_name) + resource_registry.register_resource_by_name(resource_name) controller = base.create_resource(collection_name, resource_name, plugin, params, allow_bulk=True, diff --git a/neutron/plugins/nec/extensions/packetfilter.py b/neutron/plugins/nec/extensions/packetfilter.py index 7c9971f8a96..3d89cf4e25a 100644 --- a/neutron/plugins/nec/extensions/packetfilter.py +++ b/neutron/plugins/nec/extensions/packetfilter.py @@ -21,7 +21,8 @@ from neutron.api.v2 import base from neutron.common import constants from neutron.common import exceptions from neutron import manager -from neutron import quota +from neutron.quota import resource as quota_resource +from neutron.quota import resource_registry quota_packet_filter_opts = [ @@ -180,10 +181,10 @@ class Packetfilter(extensions.ExtensionDescriptor): @classmethod def get_resources(cls): - qresource = quota.CountableResource(RESOURCE, - quota._count_resource, - 'quota_%s' % RESOURCE) - quota.QUOTAS.register_resource(qresource) + qresource = quota_resource.CountableResource( + RESOURCE, quota_resource._count_resource, 'quota_%s' % RESOURCE) + + resource_registry.register_resource(qresource) resource = base.create_resource(COLLECTION, RESOURCE, manager.NeutronManager.get_plugin(), diff --git a/neutron/quota/__init__.py b/neutron/quota/__init__.py index f71b14aabe1..97b466e872a 100644 --- a/neutron/quota/__init__.py +++ b/neutron/quota/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack Foundation +# Copyright (c) 2015 OpenStack Foundation. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -25,6 +25,7 @@ import webob from neutron.common import exceptions from neutron.i18n import _LI, _LW +from neutron.quota import resource_registry LOG = logging.getLogger(__name__) @@ -33,6 +34,7 @@ QUOTA_DB_DRIVER = '%s.DbQuotaDriver' % QUOTA_DB_MODULE QUOTA_CONF_DRIVER = 'neutron.quota.ConfDriver' default_quota_items = ['network', 'subnet', 'port'] + quota_opts = [ cfg.ListOpt('quota_items', default=default_quota_items, @@ -59,6 +61,11 @@ quota_opts = [ cfg.StrOpt('quota_driver', default=QUOTA_DB_DRIVER, help=_('Default driver to use for quota checks')), + cfg.BoolOpt('track_quota_usage', + default=True, + help=_('Keep in track in the database of current resource' + 'quota usage. Plugins which do not leverage the ' + 'neutron database should set this flag to False')), ] # Register the configuration options cfg.CONF.register_opts(quota_opts, 'QUOTAS') @@ -146,67 +153,19 @@ class ConfDriver(object): raise webob.exc.HTTPForbidden(msg) -class BaseResource(object): - """Describe a single resource for quota checking.""" - - def __init__(self, name, flag): - """Initializes a resource. - - :param name: The name of the resource, i.e., "instances". - :param flag: The name of the flag or configuration option - """ - - self.name = name - self.flag = flag - - @property - def default(self): - """Return the default value of the quota.""" - # Any negative value will be interpreted as an infinite quota, - # and stored as -1 for compatibility with current behaviour - value = getattr(cfg.CONF.QUOTAS, - self.flag, - cfg.CONF.QUOTAS.default_quota) - return max(value, -1) - - -class CountableResource(BaseResource): - """Describe a resource where the counts are determined by a function.""" - - def __init__(self, name, count, flag=None): - """Initializes a CountableResource. - - Countable resources are those resources which directly - correspond to objects in the database, i.e., netowk, subnet, - etc.,. A CountableResource must be constructed with a counting - function, which will be called to determine the current counts - of the resource. - - The counting function will be passed the context, along with - the extra positional and keyword arguments that are passed to - Quota.count(). It should return an integer specifying the - count. - - :param name: The name of the resource, i.e., "instances". - :param count: A callable which returns the count of the - resource. The arguments passed are as described - above. - :param flag: The name of the flag or configuration option - which specifies the default value of the quota - for this resource. - """ - - super(CountableResource, self).__init__(name, flag=flag) - self.count = count - - class QuotaEngine(object): """Represent the set of recognized quotas.""" + _instance = None + + @classmethod + def get_instance(cls): + if not cls._instance: + cls._instance = cls() + return cls._instance + def __init__(self, quota_driver_class=None): """Initialize a Quota object.""" - - self._resources = {} self._driver = None self._driver_class = quota_driver_class @@ -232,29 +191,7 @@ class QuotaEngine(object): LOG.info(_LI('Loaded quota_driver: %s.'), _driver_class) return self._driver - def __contains__(self, resource): - return resource in self._resources - - def register_resource(self, resource): - """Register a resource.""" - if resource.name in self._resources: - LOG.warn(_LW('%s is already registered.'), resource.name) - return - self._resources[resource.name] = resource - - def register_resource_by_name(self, resourcename): - """Register a resource by name.""" - resource = CountableResource(resourcename, _count_resource, - 'quota_' + resourcename) - self.register_resource(resource) - - def register_resources(self, resources): - """Register a list of resources.""" - - for resource in resources: - self.register_resource(resource) - - def count(self, context, resource, *args, **kwargs): + def count(self, context, resource_name, *args, **kwargs): """Count a resource. For countable resources, invokes the count() function and @@ -263,13 +200,13 @@ class QuotaEngine(object): the resource. :param context: The request context, for access checks. - :param resource: The name of the resource, as a string. + :param resource_name: The name of the resource, as a string. """ # Get the resource - res = self._resources.get(resource) + res = resource_registry.get_resource(resource_name) if not res or not hasattr(res, 'count'): - raise exceptions.QuotaResourceUnknown(unknown=[resource]) + raise exceptions.QuotaResourceUnknown(unknown=[resource_name]) return res.count(context, *args, **kwargs) @@ -297,7 +234,8 @@ class QuotaEngine(object): """ # Verify that resources are managed by the quota engine requested_resources = set(values.keys()) - managed_resources = set([res for res in self._resources.keys() + managed_resources = set([res for res in + resource_registry.get_all_resources() if res in requested_resources]) # Make sure we accounted for all of them... @@ -306,31 +244,11 @@ class QuotaEngine(object): raise exceptions.QuotaResourceUnknown( unknown=sorted(unknown_resources)) - return self.get_driver().limit_check(context, tenant_id, - self._resources, values) - - @property - def resources(self): - return self._resources + return self.get_driver().limit_check( + context, tenant_id, resource_registry.get_all_resources(), values) -QUOTAS = QuotaEngine() - - -def _count_resource(context, plugin, resources, tenant_id): - count_getter_name = "get_%s_count" % resources - - # Some plugins support a count method for particular resources, - # using a DB's optimized counting features. We try to use that one - # if present. Otherwise just use regular getter to retrieve all objects - # and count in python, allowing older plugins to still be supported - try: - obj_count_getter = getattr(plugin, count_getter_name) - return obj_count_getter(context, filters={'tenant_id': [tenant_id]}) - except (NotImplementedError, AttributeError): - obj_getter = getattr(plugin, "get_%s" % resources) - obj_list = obj_getter(context, filters={'tenant_id': [tenant_id]}) - return len(obj_list) if obj_list else 0 +QUOTAS = QuotaEngine.get_instance() def register_resources_from_config(): @@ -342,12 +260,9 @@ def register_resources_from_config(): "quota_items option is deprecated as of Liberty." "Resource REST controllers should take care of registering " "resources with the quota engine.")) - resources = [] for resource_item in (set(cfg.CONF.QUOTAS.quota_items) - set(default_quota_items)): - resources.append(CountableResource(resource_item, _count_resource, - 'quota_' + resource_item)) - QUOTAS.register_resources(resources) + resource_registry.register_resource_by_name(resource_item) register_resources_from_config() diff --git a/neutron/quota/resource.py b/neutron/quota/resource.py new file mode 100644 index 00000000000..25bcba761bd --- /dev/null +++ b/neutron/quota/resource.py @@ -0,0 +1,241 @@ +# Copyright (c) 2015 OpenStack Foundation. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_concurrency import lockutils +from oslo_config import cfg +from oslo_db import api as oslo_db_api +from oslo_db import exception as oslo_db_exception +from oslo_log import log +from sqlalchemy import event + +from neutron.db import api as db_api +from neutron.db.quota import api as quota_api +from neutron.i18n import _LE + +LOG = log.getLogger(__name__) + + +def _count_resource(context, plugin, resources, tenant_id): + count_getter_name = "get_%s_count" % resources + + # Some plugins support a count method for particular resources, + # using a DB's optimized counting features. We try to use that one + # if present. Otherwise just use regular getter to retrieve all objects + # and count in python, allowing older plugins to still be supported + try: + obj_count_getter = getattr(plugin, count_getter_name) + meh = obj_count_getter(context, filters={'tenant_id': [tenant_id]}) + return meh + except (NotImplementedError, AttributeError): + obj_getter = getattr(plugin, "get_%s" % resources) + obj_list = obj_getter(context, filters={'tenant_id': [tenant_id]}) + return len(obj_list) if obj_list else 0 + + +class BaseResource(object): + """Describe a single resource for quota checking.""" + + def __init__(self, name, flag): + """Initializes a resource. + + :param name: The name of the resource, i.e., "instances". + :param flag: The name of the flag or configuration option + """ + + self.name = name + self.flag = flag + + @property + def default(self): + """Return the default value of the quota.""" + # Any negative value will be interpreted as an infinite quota, + # and stored as -1 for compatibility with current behaviour + value = getattr(cfg.CONF.QUOTAS, + self.flag, + cfg.CONF.QUOTAS.default_quota) + return max(value, -1) + + @property + def dirty(self): + """Return the current state of the Resource instance. + + :returns: True if the resource count is out of sync with actual date, + False if it is in sync, and None if the resource instance + does not track usage. + """ + + +class CountableResource(BaseResource): + """Describe a resource where the counts are determined by a function.""" + + def __init__(self, name, count, flag=None): + """Initializes a CountableResource. + + Countable resources are those resources which directly + correspond to objects in the database, i.e., netowk, subnet, + etc.,. A CountableResource must be constructed with a counting + function, which will be called to determine the current counts + of the resource. + + The counting function will be passed the context, along with + the extra positional and keyword arguments that are passed to + Quota.count(). It should return an integer specifying the + count. + + :param name: The name of the resource, i.e., "instances". + :param count: A callable which returns the count of the + resource. The arguments passed are as described + above. + :param flag: The name of the flag or configuration option + which specifies the default value of the quota + for this resource. + """ + + super(CountableResource, self).__init__(name, flag=flag) + self.count = count + + +class TrackedResource(BaseResource): + """Resource which keeps track of its usage data.""" + + def __init__(self, name, model_class, flag): + """Initializes an instance for a given resource. + + TrackedResource are directly mapped to data model classes. + Resource usage is tracked in the database, and the model class to + which this resource refers is monitored to ensure always "fresh" + usage data are employed when performing quota checks. + + This class operates under the assumption that the model class + describing the resource has a tenant identifier attribute. + + :param name: The name of the resource, i.e., "networks". + :param model_class: The sqlalchemy model class of the resource for + which this instance is being created + :param flag: The name of the flag or configuration option + which specifies the default value of the quota + for this resource. + """ + super(TrackedResource, self).__init__(name, flag) + # Register events for addition/removal of records in the model class + # As tenant_id is immutable for all Neutron objects there is no need + # to register a listener for update events + self._model_class = model_class + self._dirty_tenants = set() + self._out_of_sync_tenants = set() + + @property + def dirty(self): + return self._dirty_tenants + + @lockutils.synchronized('dirty_tenants') + def mark_dirty(self, context, nested=False): + if not self._dirty_tenants: + return + with context.session.begin(nested=nested, subtransactions=True): + for tenant_id in self._dirty_tenants: + quota_api.set_quota_usage_dirty(context, self.name, tenant_id) + LOG.debug(("Persisted dirty status for tenant:%(tenant_id)s " + "on resource:%(resource)s"), + {'tenant_id': tenant_id, 'resource': self.name}) + self._out_of_sync_tenants |= self._dirty_tenants + self._dirty_tenants.clear() + + @lockutils.synchronized('dirty_tenants') + def _db_event_handler(self, mapper, _conn, target): + tenant_id = target.get('tenant_id') + if not tenant_id: + # NOTE: This is an unexpected error condition. Log anomaly but do + # not raise as this might have unexpected effects on other + # operations + LOG.error(_LE("Model class %s does not have tenant_id attribute"), + target) + return + self._dirty_tenants.add(tenant_id) + + # Retry the operation if a duplicate entry exception is raised. This + # can happen is two or more workers are trying to create a resource of a + # give kind for the same tenant concurrently. Retrying the operation will + # ensure that an UPDATE statement is emitted rather than an INSERT one + @oslo_db_api.wrap_db_retry( + max_retries=db_api.MAX_RETRIES, + exception_checker=lambda exc: + isinstance(exc, oslo_db_exception.DBDuplicateEntry)) + def _set_quota_usage(self, context, tenant_id, in_use): + return quota_api.set_quota_usage(context, self.name, + tenant_id, in_use=in_use) + + def _resync(self, context, tenant_id, in_use): + # Update quota usage + usage_info = self._set_quota_usage( + context, tenant_id, in_use=in_use) + self._dirty_tenants.discard(tenant_id) + self._out_of_sync_tenants.discard(tenant_id) + LOG.debug(("Unset dirty status for tenant:%(tenant_id)s on " + "resource:%(resource)s"), + {'tenant_id': tenant_id, 'resource': self.name}) + return usage_info + + def resync(self, context, tenant_id): + if tenant_id not in self._out_of_sync_tenants: + return + LOG.debug(("Synchronizing usage tracker for tenant:%(tenant_id)s on " + "resource:%(resource)s"), + {'tenant_id': tenant_id, 'resource': self.name}) + in_use = context.session.query(self._model_class).filter_by( + tenant_id=tenant_id).count() + # Update quota usage + return self._resync(context, tenant_id, in_use) + + def count(self, context, _plugin, _resources, tenant_id, + resync_usage=False): + """Return the current usage count for the resource.""" + # Load current usage data + usage_info = quota_api.get_quota_usage_by_resource_and_tenant( + context, self.name, tenant_id) + # If dirty or missing, calculate actual resource usage querying + # the database and set/create usage info data + # NOTE: this routine "trusts" usage counters at service startup. This + # assumption is generally valid, but if the database is tampered with, + # or if data migrations do not take care of usage counters, the + # assumption will not hold anymore + if (tenant_id in self._dirty_tenants or not usage_info + or usage_info.dirty): + LOG.debug(("Usage tracker for resource:%(resource)s and tenant:" + "%(tenant_id)s is out of sync, need to count used " + "quota"), {'resource': self.name, + 'tenant_id': tenant_id}) + in_use = context.session.query(self._model_class).filter_by( + tenant_id=tenant_id).count() + # Update quota usage, if requested (by default do not do that, as + # typically one counts before adding a record, and that would mark + # the usage counter as dirty again) + if resync_usage or not usage_info: + usage_info = self._resync(context, tenant_id, in_use) + else: + usage_info = quota_api.QuotaUsageInfo(usage_info.resource, + usage_info.tenant_id, + in_use, + usage_info.reserved, + usage_info.dirty) + + return usage_info.total + + def register_events(self): + event.listen(self._model_class, 'after_insert', self._db_event_handler) + event.listen(self._model_class, 'after_delete', self._db_event_handler) + + def unregister_events(self): + event.remove(self._model_class, 'after_insert', self._db_event_handler) + event.remove(self._model_class, 'after_delete', self._db_event_handler) diff --git a/neutron/quota/resource_registry.py b/neutron/quota/resource_registry.py new file mode 100644 index 00000000000..a0dfeebe7dc --- /dev/null +++ b/neutron/quota/resource_registry.py @@ -0,0 +1,245 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_log import log +import six + +from neutron.i18n import _LI, _LW +from neutron.quota import resource + +LOG = log.getLogger(__name__) + + +# Wrappers for easing access to the ResourceRegistry singleton + + +def register_resource(resource): + ResourceRegistry.get_instance().register_resource(resource) + + +def register_resource_by_name(resource_name): + ResourceRegistry.get_instance().register_resource_by_name(resource_name) + + +def get_all_resources(): + return ResourceRegistry.get_instance().resources + + +def get_resource(resource_name): + return ResourceRegistry.get_instance().get_resource(resource_name) + + +def is_tracked(resource_name): + return ResourceRegistry.get_instance().is_tracked(resource_name) + + +# auxiliary functions and decorators + + +def set_resources_dirty(context): + """Sets the dirty bit for resources with usage changes. + + This routine scans all registered resources, and, for those whose + dirty status is True, sets the dirty bit to True in the database + for the appropriate tenants. + + Please note that this routine begins a nested transaction, and it + is not recommended that this transaction begins within another + transaction. For this reason the function will raise a SqlAlchemy + exception if such an attempt is made. + + :param context: a Neutron request context with a DB session + """ + if not cfg.CONF.QUOTAS.track_quota_usage: + return + + for res in get_all_resources().values(): + with context.session.begin(): + if is_tracked(res.name) and res.dirty: + res.mark_dirty(context, nested=True) + + +def resync_resource(context, resource_name, tenant_id): + if not cfg.CONF.QUOTAS.track_quota_usage: + return + + if is_tracked(resource_name): + res = get_resource(resource_name) + # If the resource is tracked count supports the resync_usage parameter + res.resync(context, tenant_id) + + +def mark_resources_dirty(f): + """Decorator for functions which alter resource usage. + + This decorator ensures set_resource_dirty is invoked after completion + of the decorated function. + """ + + @six.wraps(f) + def wrapper(_self, context, *args, **kwargs): + ret_val = f(_self, context, *args, **kwargs) + set_resources_dirty(context) + return ret_val + + return wrapper + + +class tracked_resources(object): + """Decorator for specifying resources for which usage should be tracked. + + A plugin class can use this decorator to specify for which resources + usage info should be tracked into an appropriate table rather than being + explicitly counted. + """ + + def __init__(self, override=False, **kwargs): + self._tracked_resources = kwargs + self._override = override + + def __call__(self, f): + + @six.wraps(f) + def wrapper(*args, **kwargs): + registry = ResourceRegistry.get_instance() + for resource_name in self._tracked_resources: + registry.set_tracked_resource( + resource_name, + self._tracked_resources[resource_name], + self._override) + return f(*args, **kwargs) + + return wrapper + + +class ResourceRegistry(object): + """Registry for resource subject to quota limits. + + This class keeps track of Neutron resources for which quota limits are + enforced, regardless of whether their usage is being tracked or counted. + + For tracked-usage resources, that is to say those resources for which + there are usage counters which are kept in sync with the actual number + of rows in the database, this class allows the plugin to register their + names either explicitly or through the @tracked_resources decorator, + which should preferrably be applied to the __init__ method of the class. + """ + + _instance = None + + @classmethod + def get_instance(cls): + if cls._instance is None: + cls._instance = cls() + return cls._instance + + def __init__(self): + self._resources = {} + # Map usage tracked resources to the correspondent db model class + self._tracked_resource_mappings = {} + + def __contains__(self, resource): + return resource in self._resources + + def _create_resource_instance(self, resource_name): + """Factory function for quota Resource. + + This routine returns a resource instance of the appropriate type + according to system configuration. + + If QUOTAS.track_quota_usage is True, and there is a model mapping for + the current resource, this function will return an instance of + AccountedResource; otherwise an instance of CountableResource. + """ + + if (not cfg.CONF.QUOTAS.track_quota_usage or + resource_name not in self._tracked_resource_mappings): + LOG.info(_LI("Creating instance of CountableResource for " + "resource:%s"), resource_name) + return resource.CountableResource( + resource_name, resource._count_resource, + 'quota_%s' % resource_name) + else: + LOG.info(_LI("Creating instance of TrackedResource for " + "resource:%s"), resource_name) + return resource.TrackedResource( + resource_name, + self._tracked_resource_mappings[resource_name], + 'quota_%s' % resource_name) + + def set_tracked_resource(self, resource_name, model_class, override=False): + # Do not do anything if tracking is disabled by config + if not cfg.CONF.QUOTAS.track_quota_usage: + return + + current_model_class = self._tracked_resource_mappings.setdefault( + resource_name, model_class) + + # Check whether setdefault also set the entry in the dict + if current_model_class != model_class: + LOG.debug("A model class is already defined for %(resource)s: " + "%(current_model_class)s. Override:%(override)s", + {'resource': resource_name, + 'current_model_class': current_model_class, + 'override': override}) + if override: + self._tracked_resource_mappings[resource_name] = model_class + LOG.debug("Tracking information for resource: %s configured", + resource_name) + + def is_tracked(self, resource_name): + """Find out if a resource if tracked or not. + + :param resource_name: name of the resource. + :returns True if resource_name is registered and tracked, otherwise + False. Please note that here when False it returned it + simply means that resource_name is not a TrackedResource + instance, it does not necessarily mean that the resource + is not registered. + """ + return resource_name in self._tracked_resource_mappings + + def register_resource(self, resource): + if resource.name in self._resources: + LOG.warn(_LW('%s is already registered'), resource.name) + if resource.name in self._tracked_resource_mappings: + resource.register_events() + self._resources[resource.name] = resource + + def register_resources(self, resources): + for res in resources: + self.register_resource(res) + + def register_resource_by_name(self, resource_name): + """Register a resource by name.""" + resource = self._create_resource_instance(resource_name) + self.register_resource(resource) + + def unregister_resources(self): + """Unregister all resources.""" + for (res_name, res) in self._resources.items(): + if res_name in self._tracked_resource_mappings: + res.unregister_events() + self._resources.clear() + self._tracked_resource_mappings.clear() + + def get_resource(self, resource_name): + """Return a resource given its name. + + :returns: The resource instance or None if the resource is not found + """ + return self._resources.get(resource_name) + + @property + def resources(self): + return self._resources diff --git a/neutron/tests/unit/api/v2/test_base.py b/neutron/tests/unit/api/v2/test_base.py index eae4dd21b53..0ee9c2ec313 100644 --- a/neutron/tests/unit/api/v2/test_base.py +++ b/neutron/tests/unit/api/v2/test_base.py @@ -37,6 +37,7 @@ from neutron import context from neutron import manager from neutron import policy from neutron import quota +from neutron.quota import resource_registry from neutron.tests import base from neutron.tests import fake_notifier from neutron.tests import tools @@ -1289,6 +1290,12 @@ class NotificationTest(APIv2TestBase): class DHCPNotificationTest(APIv2TestBase): + + def setUp(self): + # This test does not have database support so tracking cannot be used + cfg.CONF.set_override('track_quota_usage', False, group='QUOTAS') + super(DHCPNotificationTest, self).setUp() + def _test_dhcp_notifier(self, opname, resource, initial_input=None): instance = self.plugin.return_value instance.get_networks.return_value = initial_input @@ -1340,6 +1347,23 @@ class DHCPNotificationTest(APIv2TestBase): class QuotaTest(APIv2TestBase): + + def setUp(self): + # This test does not have database support so tracking cannot be used + cfg.CONF.set_override('track_quota_usage', False, group='QUOTAS') + super(QuotaTest, self).setUp() + # Use mock to let the API use a different QuotaEngine instance for + # unit test in this class. This will ensure resource are registered + # again and instanciated with neutron.quota.resource.CountableResource + replacement_registry = resource_registry.ResourceRegistry() + registry_patcher = mock.patch('neutron.quota.resource_registry.' + 'ResourceRegistry.get_instance') + mock_registry = registry_patcher.start().return_value + mock_registry.get_resource = replacement_registry.get_resource + mock_registry.resources = replacement_registry.resources + # Register a resource + replacement_registry.register_resource_by_name('network') + def test_create_network_quota(self): cfg.CONF.set_override('quota_network', 1, group='QUOTAS') initial_input = {'network': {'name': 'net1', 'tenant_id': _uuid()}} @@ -1384,9 +1408,10 @@ class QuotaTest(APIv2TestBase): class ExtensionTestCase(base.BaseTestCase): def setUp(self): + # This test does not have database support so tracking cannot be used + cfg.CONF.set_override('track_quota_usage', False, group='QUOTAS') super(ExtensionTestCase, self).setUp() plugin = 'neutron.neutron_plugin_base_v2.NeutronPluginBaseV2' - # Ensure existing ExtensionManager is not used extensions.PluginAwareExtensionManager._instance = None diff --git a/neutron/tests/unit/extensions/extensionattribute.py b/neutron/tests/unit/extensions/extensionattribute.py index f289c8b0625..dcf2c8c2385 100644 --- a/neutron/tests/unit/extensions/extensionattribute.py +++ b/neutron/tests/unit/extensions/extensionattribute.py @@ -18,7 +18,7 @@ import abc from neutron.api import extensions from neutron.api.v2 import base from neutron import manager -from neutron import quota +from neutron.quota import resource_registry # Attribute Map @@ -69,7 +69,7 @@ class Extensionattribute(extensions.ExtensionDescriptor): collection_name = resource_name + "s" params = RESOURCE_ATTRIBUTE_MAP.get(collection_name, dict()) - quota.QUOTAS.register_resource_by_name(resource_name) + resource_registry.register_resource_by_name(resource_name) controller = base.create_resource(collection_name, resource_name, diff --git a/neutron/tests/unit/extensions/test_quotasv2.py b/neutron/tests/unit/extensions/test_quotasv2.py index bf1ac304cae..e0780e1ee78 100644 --- a/neutron/tests/unit/extensions/test_quotasv2.py +++ b/neutron/tests/unit/extensions/test_quotasv2.py @@ -29,6 +29,7 @@ from neutron.common import exceptions from neutron import context from neutron.db.quota import driver from neutron import quota +from neutron.quota import resource_registry from neutron.tests import base from neutron.tests import tools from neutron.tests.unit.api.v2 import test_base @@ -64,7 +65,7 @@ class QuotaExtensionTestCase(testlib_api.WebTestCase): self.plugin.return_value.supported_extension_aliases = ['quotas'] # QUOTAS will register the items in conf when starting # extra1 here is added later, so have to do it manually - quota.QUOTAS.register_resource_by_name('extra1') + resource_registry.register_resource_by_name('extra1') ext_mgr = extensions.PluginAwareExtensionManager.get_instance() app = config.load_paste_app('extensions_test_app') ext_middleware = extensions.ExtensionMiddleware(app, ext_mgr=ext_mgr) diff --git a/neutron/tests/unit/quota/__init__.py b/neutron/tests/unit/quota/__init__.py new file mode 100644 index 00000000000..3fd44693afd --- /dev/null +++ b/neutron/tests/unit/quota/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2015 OpenStack Foundation. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sa + +from neutron.db import model_base +from neutron.db import models_v2 + +# Model classes for test resources + + +class MehModel(model_base.BASEV2, models_v2.HasTenant): + meh = sa.Column(sa.String(8), primary_key=True) + + +class OtherMehModel(model_base.BASEV2, models_v2.HasTenant): + othermeh = sa.Column(sa.String(8), primary_key=True) diff --git a/neutron/tests/unit/quota/test_resource.py b/neutron/tests/unit/quota/test_resource.py new file mode 100644 index 00000000000..d1e890cd080 --- /dev/null +++ b/neutron/tests/unit/quota/test_resource.py @@ -0,0 +1,232 @@ +# Copyright (c) 2015 OpenStack Foundation. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import random + +import mock +from oslo_config import cfg + +from neutron import context +from neutron.db import api as db_api +from neutron.db.quota import api as quota_api +from neutron.quota import resource +from neutron.tests import base +from neutron.tests.unit import quota as test_quota +from neutron.tests.unit import testlib_api + + +meh_quota_flag = 'quota_meh' +meh_quota_opts = [cfg.IntOpt(meh_quota_flag, default=99)] +random.seed() + + +class TestTrackedResource(testlib_api.SqlTestCaseLight): + + def _add_data(self, tenant_id=None): + session = db_api.get_session() + with session.begin(): + tenant_id = tenant_id or self.tenant_id + session.add(test_quota.MehModel( + meh='meh_%d' % random.randint(0, 10000), + tenant_id=tenant_id)) + session.add(test_quota.MehModel( + meh='meh_%d' % random.randint(0, 10000), + tenant_id=tenant_id)) + + def _delete_data(self): + session = db_api.get_session() + with session.begin(): + query = session.query(test_quota.MehModel).filter_by( + tenant_id=self.tenant_id) + for item in query: + session.delete(item) + + def _update_data(self): + session = db_api.get_session() + with session.begin(): + query = session.query(test_quota.MehModel).filter_by( + tenant_id=self.tenant_id) + for item in query: + item['meh'] = 'meh-%s' % item['meh'] + session.add(item) + + def setUp(self): + base.BaseTestCase.config_parse() + cfg.CONF.register_opts(meh_quota_opts, 'QUOTAS') + self.addCleanup(cfg.CONF.reset) + self.resource = 'meh' + self.other_resource = 'othermeh' + self.tenant_id = 'meh' + self.context = context.Context( + user_id='', tenant_id=self.tenant_id, is_admin=False) + super(TestTrackedResource, self).setUp() + + def _register_events(self, res): + res.register_events() + self.addCleanup(res.unregister_events) + + def _create_resource(self): + res = resource.TrackedResource( + self.resource, test_quota.MehModel, meh_quota_flag) + self._register_events(res) + return res + + def _create_other_resource(self): + res = resource.TrackedResource( + self.other_resource, test_quota.OtherMehModel, meh_quota_flag) + self._register_events(res) + return res + + def test_count_first_call_with_dirty_false(self): + quota_api.set_quota_usage( + self.context, self.resource, self.tenant_id, in_use=1) + res = self._create_resource() + self._add_data() + # explicitly set dirty flag to False + quota_api.set_all_quota_usage_dirty( + self.context, self.resource, dirty=False) + # Expect correct count to be returned anyway since the first call to + # count() always resyncs with the db + self.assertEqual(2, res.count(self.context, + None, None, + self.tenant_id)) + + def _test_count(self): + res = self._create_resource() + quota_api.set_quota_usage( + self.context, res.name, self.tenant_id, in_use=0) + self._add_data() + return res + + def test_count_with_dirty_false(self): + res = self._test_count() + res.count(self.context, None, None, self.tenant_id) + # At this stage count has been invoked, and the dirty flag should be + # false. Another invocation of count should not query the model class + set_quota = 'neutron.db.quota.api.set_quota_usage' + with mock.patch(set_quota) as mock_set_quota: + self.assertEqual(0, mock_set_quota.call_count) + self.assertEqual(2, res.count(self.context, + None, None, + self.tenant_id)) + + def test_count_with_dirty_true_resync(self): + res = self._test_count() + # Expect correct count to be returned, which also implies + # set_quota_usage has been invoked with the correct parameters + self.assertEqual(2, res.count(self.context, + None, None, + self.tenant_id, + resync_usage=True)) + + def test_count_with_dirty_true_resync_calls_set_quota_usage(self): + res = self._test_count() + set_quota_usage = 'neutron.db.quota.api.set_quota_usage' + with mock.patch(set_quota_usage) as mock_set_quota_usage: + quota_api.set_quota_usage_dirty(self.context, + self.resource, + self.tenant_id) + res.count(self.context, None, None, self.tenant_id, + resync_usage=True) + mock_set_quota_usage.assert_called_once_with( + self.context, self.resource, self.tenant_id, in_use=2) + + def test_count_with_dirty_true_no_usage_info(self): + res = self._create_resource() + self._add_data() + # Invoke count without having usage info in DB - Expect correct + # count to be returned + self.assertEqual(2, res.count(self.context, + None, None, + self.tenant_id)) + + def test_count_with_dirty_true_no_usage_info_calls_set_quota_usage(self): + res = self._create_resource() + self._add_data() + set_quota_usage = 'neutron.db.quota.api.set_quota_usage' + with mock.patch(set_quota_usage) as mock_set_quota_usage: + quota_api.set_quota_usage_dirty(self.context, + self.resource, + self.tenant_id) + res.count(self.context, None, None, self.tenant_id, + resync_usage=True) + mock_set_quota_usage.assert_called_once_with( + self.context, self.resource, self.tenant_id, in_use=2) + + def test_add_delete_data_triggers_event(self): + res = self._create_resource() + other_res = self._create_other_resource() + # Validate dirty tenants since mock does not work well with sqlalchemy + # event handlers. + self._add_data() + self._add_data('someone_else') + self.assertEqual(2, len(res._dirty_tenants)) + # Also, the dirty flag should not be set for other resources + self.assertEqual(0, len(other_res._dirty_tenants)) + self.assertIn(self.tenant_id, res._dirty_tenants) + self.assertIn('someone_else', res._dirty_tenants) + + def test_delete_data_triggers_event(self): + res = self._create_resource() + self._add_data() + self._add_data('someone_else') + # Artificially clear _dirty_tenants + res._dirty_tenants.clear() + self._delete_data() + # We did not delete "someone_else", so expect only a single dirty + # tenant + self.assertEqual(1, len(res._dirty_tenants)) + self.assertIn(self.tenant_id, res._dirty_tenants) + + def test_update_does_not_trigger_event(self): + res = self._create_resource() + self._add_data() + self._add_data('someone_else') + # Artificially clear _dirty_tenants + res._dirty_tenants.clear() + self._update_data() + self.assertEqual(0, len(res._dirty_tenants)) + + def test_mark_dirty(self): + res = self._create_resource() + self._add_data() + self._add_data('someone_else') + set_quota_usage = 'neutron.db.quota.api.set_quota_usage_dirty' + with mock.patch(set_quota_usage) as mock_set_quota_usage: + res.mark_dirty(self.context) + self.assertEqual(2, mock_set_quota_usage.call_count) + mock_set_quota_usage.assert_any_call( + self.context, self.resource, self.tenant_id) + mock_set_quota_usage.assert_any_call( + self.context, self.resource, 'someone_else') + + def test_mark_dirty_no_dirty_tenant(self): + res = self._create_resource() + set_quota_usage = 'neutron.db.quota.api.set_quota_usage_dirty' + with mock.patch(set_quota_usage) as mock_set_quota_usage: + res.mark_dirty(self.context) + self.assertFalse(mock_set_quota_usage.call_count) + + def test_resync(self): + res = self._create_resource() + self._add_data() + res.mark_dirty(self.context) + # self.tenant_id now is out of sync + set_quota_usage = 'neutron.db.quota.api.set_quota_usage' + with mock.patch(set_quota_usage) as mock_set_quota_usage: + res.resync(self.context, self.tenant_id) + # and now it should be in sync + self.assertNotIn(self.tenant_id, res._out_of_sync_tenants) + mock_set_quota_usage.assert_called_once_with( + self.context, self.resource, self.tenant_id, in_use=2) diff --git a/neutron/tests/unit/quota/test_resource_registry.py b/neutron/tests/unit/quota/test_resource_registry.py new file mode 100644 index 00000000000..6d1d272060f --- /dev/null +++ b/neutron/tests/unit/quota/test_resource_registry.py @@ -0,0 +1,159 @@ +# Copyright (c) 2015 OpenStack Foundation. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from oslo_config import cfg + +from neutron import context +from neutron.quota import resource +from neutron.quota import resource_registry +from neutron.tests import base +from neutron.tests.unit import quota as test_quota + + +class TestResourceRegistry(base.DietTestCase): + + def setUp(self): + super(TestResourceRegistry, self).setUp() + self.registry = resource_registry.ResourceRegistry.get_instance() + # clean up the registry at every test + self.registry.unregister_resources() + + def test_set_tracked_resource_new_resource(self): + self.registry.set_tracked_resource('meh', test_quota.MehModel) + self.assertEqual(test_quota.MehModel, + self.registry._tracked_resource_mappings['meh']) + + def test_set_tracked_resource_existing_with_override(self): + self.test_set_tracked_resource_new_resource() + self.registry.set_tracked_resource('meh', test_quota.OtherMehModel, + override=True) + # Overidde is set to True, the model class should change + self.assertEqual(test_quota.OtherMehModel, + self.registry._tracked_resource_mappings['meh']) + + def test_set_tracked_resource_existing_no_override(self): + self.test_set_tracked_resource_new_resource() + self.registry.set_tracked_resource('meh', test_quota.OtherMehModel) + # Overidde is set to false, the model class should not change + self.assertEqual(test_quota.MehModel, + self.registry._tracked_resource_mappings['meh']) + + def _test_register_resource_by_name(self, resource_name, expected_type): + self.assertNotIn(resource_name, self.registry._resources) + self.registry.register_resource_by_name(resource_name) + self.assertIn(resource_name, self.registry._resources) + self.assertIsInstance(self.registry.get_resource(resource_name), + expected_type) + + def test_register_resource_by_name_tracked(self): + self.test_set_tracked_resource_new_resource() + self._test_register_resource_by_name('meh', resource.TrackedResource) + + def test_register_resource_by_name_not_tracked(self): + self._test_register_resource_by_name('meh', resource.CountableResource) + + def test_register_resource_by_name_with_tracking_disabled_by_config(self): + cfg.CONF.set_override('track_quota_usage', False, + group='QUOTAS') + # DietTestCase does not automatically cleans configuration overrides + self.addCleanup(cfg.CONF.reset) + self.registry.set_tracked_resource('meh', test_quota.MehModel) + self.assertNotIn( + 'meh', self.registry._tracked_resource_mappings) + self._test_register_resource_by_name('meh', resource.CountableResource) + + +class TestAuxiliaryFunctions(base.DietTestCase): + + def setUp(self): + super(TestAuxiliaryFunctions, self).setUp() + self.registry = resource_registry.ResourceRegistry.get_instance() + # clean up the registry at every test + self.registry.unregister_resources() + + def test_resync_tracking_disabled(self): + cfg.CONF.set_override('track_quota_usage', False, + group='QUOTAS') + # DietTestCase does not automatically cleans configuration overrides + self.addCleanup(cfg.CONF.reset) + with mock.patch('neutron.quota.resource.' + 'TrackedResource.resync') as mock_resync: + self.registry.set_tracked_resource('meh', test_quota.MehModel) + self.registry.register_resource_by_name('meh') + resource_registry.resync_resource(mock.ANY, 'meh', 'tenant_id') + self.assertEqual(0, mock_resync.call_count) + + def test_resync_tracked_resource(self): + with mock.patch('neutron.quota.resource.' + 'TrackedResource.resync') as mock_resync: + self.registry.set_tracked_resource('meh', test_quota.MehModel) + self.registry.register_resource_by_name('meh') + resource_registry.resync_resource(mock.ANY, 'meh', 'tenant_id') + mock_resync.assert_called_once_with(mock.ANY, 'tenant_id') + + def test_resync_non_tracked_resource(self): + with mock.patch('neutron.quota.resource.' + 'TrackedResource.resync') as mock_resync: + self.registry.register_resource_by_name('meh') + resource_registry.resync_resource(mock.ANY, 'meh', 'tenant_id') + self.assertEqual(0, mock_resync.call_count) + + def test_set_resources_dirty_invoked_with_tracking_disabled(self): + cfg.CONF.set_override('track_quota_usage', False, + group='QUOTAS') + # DietTestCase does not automatically cleans configuration overrides + self.addCleanup(cfg.CONF.reset) + with mock.patch('neutron.quota.resource.' + 'TrackedResource.mark_dirty') as mock_mark_dirty: + self.registry.set_tracked_resource('meh', test_quota.MehModel) + self.registry.register_resource_by_name('meh') + resource_registry.set_resources_dirty(mock.ANY) + self.assertEqual(0, mock_mark_dirty.call_count) + + def test_set_resources_dirty_no_dirty_resource(self): + ctx = context.Context('user_id', 'tenant_id', + is_admin=False, is_advsvc=False) + with mock.patch('neutron.quota.resource.' + 'TrackedResource.mark_dirty') as mock_mark_dirty: + self.registry.set_tracked_resource('meh', test_quota.MehModel) + self.registry.register_resource_by_name('meh') + res = self.registry.get_resource('meh') + # This ensures dirty is false + res._dirty_tenants.clear() + resource_registry.set_resources_dirty(ctx) + self.assertEqual(0, mock_mark_dirty.call_count) + + def test_set_resources_dirty_no_tracked_resource(self): + ctx = context.Context('user_id', 'tenant_id', + is_admin=False, is_advsvc=False) + with mock.patch('neutron.quota.resource.' + 'TrackedResource.mark_dirty') as mock_mark_dirty: + self.registry.register_resource_by_name('meh') + resource_registry.set_resources_dirty(ctx) + self.assertEqual(0, mock_mark_dirty.call_count) + + def test_set_resources_dirty(self): + ctx = context.Context('user_id', 'tenant_id', + is_admin=False, is_advsvc=False) + with mock.patch('neutron.quota.resource.' + 'TrackedResource.mark_dirty') as mock_mark_dirty: + self.registry.set_tracked_resource('meh', test_quota.MehModel) + self.registry.register_resource_by_name('meh') + res = self.registry.get_resource('meh') + # This ensures dirty is true + res._dirty_tenants.add('tenant_id') + resource_registry.set_resources_dirty(ctx) + mock_mark_dirty.assert_called_once_with(ctx, nested=True)