openstack
/
neutron
Code Issues Proposed changes

Merge "Introduce usage data tracking for Neutron"

This commit is contained in:
Jenkins 2015-07-31 06:57:44 +00:00 committed by Gerrit Code Review
parent e40d23e26d 3c584084c3
commit 68cc0b32db
18 changed files with 1013 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
neutron/api/rpc/handlers/dhcp_rpc.py
View File

@ -29,7 +29,7 @@ from neutron.common import utils
from neutron.extensions import portbindings
from neutron.i18n import _LW
from neutron import manager
from neutron.quota import resource_registry
LOG = logging.getLogger(__name__)
@ -203,6 +203,7 @@ class DhcpRpcCallback(object):
LOG.warning(_LW('Updating lease expiration is now deprecated. Issued '
'from host %s.'), host)
@resource_registry.mark_resources_dirty
def create_dhcp_port(self, context, **kwargs):
"""Create and return dhcp port information.

29
neutron/api/v2/base.py
View File

@ -34,6 +34,7 @@ from neutron.db import api as db_api
from neutron.i18n import _LE, _LI
from neutron import policy
from neutron import quota
from neutron.quota import resource_registry
LOG = logging.getLogger(__name__)
@ -207,7 +208,15 @@ class Controller(object):
name,
resource,
pluralized=self._collection)
return getattr(self._plugin, name)(*arg_list, **kwargs)
ret_value = getattr(self._plugin, name)(*arg_list, **kwargs)
# It is simply impossible to predict whether one of this
# actions alters resource usage. For instance a tenant port
# is created when a router interface is added. Therefore it is
# important to mark as dirty resources whose counters have
# been altered by this operation
resource_registry.set_resources_dirty(request.context)
return ret_value
return _handle_action
else:
raise AttributeError()
@ -280,6 +289,9 @@ class Controller(object):
pagination_links = pagination_helper.get_links(obj_list)
if pagination_links:
collection[self._collection + "_links"] = pagination_links
# Synchronize usage trackers, if needed
resource_registry.resync_resource(
request.context, self._resource, request.context.tenant_id)
return collection
def _item(self, request, id, do_authz=False, field_list=None,
@ -436,6 +448,12 @@ class Controller(object):
**kwargs)
def notify(create_result):
# Ensure usage trackers for all resources affected by this API
# operation are marked as dirty
# TODO(salv-orlando): This operation will happen in a single
# transaction with reservation commit once that is implemented
resource_registry.set_resources_dirty(request.context)
notifier_method = self._resource + '.create.end'
self._notifier.info(request.context,
notifier_method,
@ -497,6 +515,9 @@ class Controller(object):
obj_deleter = getattr(self._plugin, action)
obj_deleter(request.context, id, **kwargs)
# A delete operation usually alters resource usage, so mark affected
# usage trackers as dirty
resource_registry.set_resources_dirty(request.context)
notifier_method = self._resource + '.delete.end'
self._notifier.info(request.context,
notifier_method,
@ -561,6 +582,12 @@ class Controller(object):
if parent_id:
kwargs[self._parent_id_name] = parent_id
obj = obj_updater(request.context, id, **kwargs)
# Usually an update operation does not alter resource usage, but as
# there might be side effects it might be worth checking for changes
# in resource usage here as well (e.g: a tenant port is created when a
# router interface is added)
resource_registry.set_resources_dirty(request.context)
result = {self._resource: self._view(request.context, obj)}
notifier_method = self._resource + '.update.end'
self._notifier.info(request.context, notifier_method, result)

4
neutron/api/v2/resource_helper.py
View File

@ -20,7 +20,7 @@ from neutron.api import extensions
from neutron.api.v2 import base
from neutron import manager
from neutron.plugins.common import constants
from neutron import quota
from neutron.quota import resource_registry
LOG = logging.getLogger(__name__)
@ -80,7 +80,7 @@ def build_resource_info(plural_mappings, resource_map, which_service,
if translate_name:
collection_name = collection_name.replace('_', '-')
if register_quota:
quota.QUOTAS.register_resource_by_name(resource_name)
resource_registry.register_resource_by_name(resource_name)
member_actions = action_map.get(resource_name, {})
controller = base.create_resource(
collection_name, resource_name, plugin, params,

4
neutron/api/v2/router.py
View File

@ -27,7 +27,7 @@ from neutron.api.v2 import attributes
from neutron.api.v2 import base
from neutron import manager
from neutron import policy
from neutron import quota
from neutron.quota import resource_registry
from neutron import wsgi
@ -106,7 +106,7 @@ class APIRouter(wsgi.Router):
_map_resource(RESOURCES[resource], resource,
attributes.RESOURCE_ATTRIBUTE_MAP.get(
RESOURCES[resource], dict()))
quota.QUOTAS.register_resource_by_name(resource)
resource_registry.register_resource_by_name(resource)
for resource in SUB_RESOURCES:
_map_resource(SUB_RESOURCES[resource]['collection_name'], resource,

2
neutron/db/ipam_backend_mixin.py
View File

@ -409,7 +409,7 @@ class IpamBackendMixin(db_base_plugin_common.DbBasePluginCommon):
enable_eagerloads(False).filter_by(id=port_id))
if not context.is_admin:
query = query.filter_by(tenant_id=context.tenant_id)
query.delete()
context.session.delete(query.first())
def _save_subnet(self, context,
network,

3
neutron/db/models_v2.py
View File

@ -133,7 +133,8 @@ class Port(model_base.BASEV2, HasId, HasTenant):
name = sa.Column(sa.String(attr.NAME_MAX_LEN))
network_id = sa.Column(sa.String(36), sa.ForeignKey("networks.id"),
nullable=False)
fixed_ips = orm.relationship(IPAllocation, backref='port', lazy='joined')
fixed_ips = orm.relationship(IPAllocation, backref='port', lazy='joined',
passive_deletes='all')
mac_address = sa.Column(sa.String(32), nullable=False)
admin_state_up = sa.Column(sa.Boolean(), nullable=False)
status = sa.Column(sa.String(16), nullable=False)

10
neutron/extensions/quotasv2.py
View File

@ -25,6 +25,7 @@ from neutron.common import constants as const
from neutron.common import exceptions as n_exc
from neutron import manager
from neutron import quota
from neutron.quota import resource_registry
from neutron import wsgi
@ -48,7 +49,7 @@ class QuotaSetsController(wsgi.Controller):
self._update_extended_attributes = True
def _update_attributes(self):
for quota_resource in QUOTAS.resources.keys():
for quota_resource in resource_registry.get_all_resources().keys():
attr_dict = EXTENDED_ATTRIBUTES_2_0[RESOURCE_COLLECTION]
attr_dict[quota_resource] = {
'allow_post': False,
@ -60,7 +61,9 @@ class QuotaSetsController(wsgi.Controller):
def _get_quotas(self, request, tenant_id):
return self._driver.get_tenant_quotas(
request.context, QUOTAS.resources, tenant_id)
request.context,
resource_registry.get_all_resources(),
tenant_id)
def create(self, request, body=None):
msg = _('POST requests are not supported on this resource.')
@ -70,7 +73,8 @@ class QuotaSetsController(wsgi.Controller):
context = request.context
self._check_admin(context)
return {self._resource_name + "s":
self._driver.get_all_quotas(context, QUOTAS.resources)}
self._driver.get_all_quotas(
context, resource_registry.get_all_resources())}
def tenant(self, request):
"""Retrieve the tenant info in context."""

4
neutron/extensions/securitygroup.py
View File

@ -26,7 +26,7 @@ from neutron.api.v2 import base
from neutron.common import constants as const
from neutron.common import exceptions as nexception
from neutron import manager
from neutron import quota
from neutron.quota import resource_registry
# Security group Exceptions
@ -305,7 +305,7 @@ class Securitygroup(extensions.ExtensionDescriptor):
for resource_name in ['security_group', 'security_group_rule']:
collection_name = resource_name.replace('_', '-') + "s"
params = RESOURCE_ATTRIBUTE_MAP.get(resource_name + "s", dict())
quota.QUOTAS.register_resource_by_name(resource_name)
resource_registry.register_resource_by_name(resource_name)
controller = base.create_resource(collection_name,
resource_name,
plugin, params, allow_bulk=True,

11
neutron/plugins/nec/extensions/packetfilter.py
View File

@ -21,7 +21,8 @@ from neutron.api.v2 import base
from neutron.common import constants
from neutron.common import exceptions
from neutron import manager
from neutron import quota
from neutron.quota import resource as quota_resource
from neutron.quota import resource_registry
quota_packet_filter_opts = [
@ -180,10 +181,10 @@ class Packetfilter(extensions.ExtensionDescriptor):
@classmethod
def get_resources(cls):
qresource = quota.CountableResource(RESOURCE,
quota._count_resource,
'quota_%s' % RESOURCE)
quota.QUOTAS.register_resource(qresource)
qresource = quota_resource.CountableResource(
RESOURCE, quota_resource._count_resource, 'quota_%s' % RESOURCE)
resource_registry.register_resource(qresource)
resource = base.create_resource(COLLECTION, RESOURCE,
manager.NeutronManager.get_plugin(),

137
neutron/quota/__init__.py
View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack Foundation
# Copyright (c) 2015 OpenStack Foundation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -25,6 +25,7 @@ import webob
from neutron.common import exceptions
from neutron.i18n import _LI, _LW
from neutron.quota import resource_registry
LOG = logging.getLogger(__name__)
@ -33,6 +34,7 @@ QUOTA_DB_DRIVER = '%s.DbQuotaDriver' % QUOTA_DB_MODULE
QUOTA_CONF_DRIVER = 'neutron.quota.ConfDriver'
default_quota_items = ['network', 'subnet', 'port']
quota_opts = [
cfg.ListOpt('quota_items',
default=default_quota_items,
@ -59,6 +61,11 @@ quota_opts = [
cfg.StrOpt('quota_driver',
default=QUOTA_DB_DRIVER,
help=_('Default driver to use for quota checks')),
cfg.BoolOpt('track_quota_usage',
default=True,
help=_('Keep in track in the database of current resource'
'quota usage. Plugins which do not leverage the '
'neutron database should set this flag to False')),
]
# Register the configuration options
cfg.CONF.register_opts(quota_opts, 'QUOTAS')
@ -146,67 +153,19 @@ class ConfDriver(object):
raise webob.exc.HTTPForbidden(msg)
class BaseResource(object):
"""Describe a single resource for quota checking."""
def __init__(self, name, flag):
"""Initializes a resource.
:param name: The name of the resource, i.e., "instances".
:param flag: The name of the flag or configuration option
"""
self.name = name
self.flag = flag
@property
def default(self):
"""Return the default value of the quota."""
# Any negative value will be interpreted as an infinite quota,
# and stored as -1 for compatibility with current behaviour
value = getattr(cfg.CONF.QUOTAS,
self.flag,
cfg.CONF.QUOTAS.default_quota)
return max(value, -1)
class CountableResource(BaseResource):
"""Describe a resource where the counts are determined by a function."""
def __init__(self, name, count, flag=None):
"""Initializes a CountableResource.
Countable resources are those resources which directly
correspond to objects in the database, i.e., netowk, subnet,
etc.,. A CountableResource must be constructed with a counting
function, which will be called to determine the current counts
of the resource.
The counting function will be passed the context, along with
the extra positional and keyword arguments that are passed to
Quota.count(). It should return an integer specifying the
count.
:param name: The name of the resource, i.e., "instances".
:param count: A callable which returns the count of the
resource. The arguments passed are as described
above.
:param flag: The name of the flag or configuration option
which specifies the default value of the quota
for this resource.
"""
super(CountableResource, self).__init__(name, flag=flag)
self.count = count
class QuotaEngine(object):
"""Represent the set of recognized quotas."""
_instance = None
@classmethod
def get_instance(cls):
if not cls._instance:
cls._instance = cls()
return cls._instance
def __init__(self, quota_driver_class=None):
"""Initialize a Quota object."""
self._resources = {}
self._driver = None
self._driver_class = quota_driver_class
@ -232,29 +191,7 @@ class QuotaEngine(object):
LOG.info(_LI('Loaded quota_driver: %s.'), _driver_class)
return self._driver
def __contains__(self, resource):
return resource in self._resources
def register_resource(self, resource):
"""Register a resource."""
if resource.name in self._resources:
LOG.warn(_LW('%s is already registered.'), resource.name)
return
self._resources[resource.name] = resource
def register_resource_by_name(self, resourcename):
"""Register a resource by name."""
resource = CountableResource(resourcename, _count_resource,
'quota_' + resourcename)
self.register_resource(resource)
def register_resources(self, resources):
"""Register a list of resources."""
for resource in resources:
self.register_resource(resource)
def count(self, context, resource, *args, **kwargs):
def count(self, context, resource_name, *args, **kwargs):
"""Count a resource.
For countable resources, invokes the count() function and
@ -263,13 +200,13 @@ class QuotaEngine(object):
the resource.
:param context: The request context, for access checks.
:param resource: The name of the resource, as a string.
:param resource_name: The name of the resource, as a string.
"""
# Get the resource
res = self._resources.get(resource)
res = resource_registry.get_resource(resource_name)
if not res or not hasattr(res, 'count'):
raise exceptions.QuotaResourceUnknown(unknown=[resource])
raise exceptions.QuotaResourceUnknown(unknown=[resource_name])
return res.count(context, *args, **kwargs)
@ -297,7 +234,8 @@ class QuotaEngine(object):
"""
# Verify that resources are managed by the quota engine
requested_resources = set(values.keys())
managed_resources = set([res for res in self._resources.keys()
managed_resources = set([res for res in
resource_registry.get_all_resources()
if res in requested_resources])
# Make sure we accounted for all of them...
@ -306,31 +244,11 @@ class QuotaEngine(object):
raise exceptions.QuotaResourceUnknown(
unknown=sorted(unknown_resources))
return self.get_driver().limit_check(context, tenant_id,
self._resources, values)
@property
def resources(self):
return self._resources
return self.get_driver().limit_check(
context, tenant_id, resource_registry.get_all_resources(), values)
QUOTAS = QuotaEngine()
def _count_resource(context, plugin, resources, tenant_id):
count_getter_name = "get_%s_count" % resources
# Some plugins support a count method for particular resources,
# using a DB's optimized counting features. We try to use that one
# if present. Otherwise just use regular getter to retrieve all objects
# and count in python, allowing older plugins to still be supported
try:
obj_count_getter = getattr(plugin, count_getter_name)
return obj_count_getter(context, filters={'tenant_id': [tenant_id]})
except (NotImplementedError, AttributeError):
obj_getter = getattr(plugin, "get_%s" % resources)
obj_list = obj_getter(context, filters={'tenant_id': [tenant_id]})
return len(obj_list) if obj_list else 0
QUOTAS = QuotaEngine.get_instance()
def register_resources_from_config():
@ -342,12 +260,9 @@ def register_resources_from_config():
"quota_items option is deprecated as of Liberty."
"Resource REST controllers should take care of registering "
"resources with the quota engine."))
resources = []
for resource_item in (set(cfg.CONF.QUOTAS.quota_items) -
set(default_quota_items)):
resources.append(CountableResource(resource_item, _count_resource,
'quota_' + resource_item))
QUOTAS.register_resources(resources)
resource_registry.register_resource_by_name(resource_item)
register_resources_from_config()

241
neutron/quota/resource.py Normal file
View File

@ -0,0 +1,241 @@
# Copyright (c) 2015 OpenStack Foundation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_concurrency import lockutils
from oslo_config import cfg
from oslo_db import api as oslo_db_api
from oslo_db import exception as oslo_db_exception
from oslo_log import log
from sqlalchemy import event
from neutron.db import api as db_api
from neutron.db.quota import api as quota_api
from neutron.i18n import _LE
LOG = log.getLogger(__name__)
def _count_resource(context, plugin, resources, tenant_id):
count_getter_name = "get_%s_count" % resources
# Some plugins support a count method for particular resources,
# using a DB's optimized counting features. We try to use that one
# if present. Otherwise just use regular getter to retrieve all objects
# and count in python, allowing older plugins to still be supported
try:
obj_count_getter = getattr(plugin, count_getter_name)
meh = obj_count_getter(context, filters={'tenant_id': [tenant_id]})
return meh
except (NotImplementedError, AttributeError):
obj_getter = getattr(plugin, "get_%s" % resources)
obj_list = obj_getter(context, filters={'tenant_id': [tenant_id]})
return len(obj_list) if obj_list else 0
class BaseResource(object):
"""Describe a single resource for quota checking."""
def __init__(self, name, flag):
"""Initializes a resource.
:param name: The name of the resource, i.e., "instances".
:param flag: The name of the flag or configuration option
"""
self.name = name
self.flag = flag
@property
def default(self):
"""Return the default value of the quota."""
# Any negative value will be interpreted as an infinite quota,
# and stored as -1 for compatibility with current behaviour
value = getattr(cfg.CONF.QUOTAS,
self.flag,
cfg.CONF.QUOTAS.default_quota)
return max(value, -1)
@property
def dirty(self):
"""Return the current state of the Resource instance.
:returns: True if the resource count is out of sync with actual date,
False if it is in sync, and None if the resource instance
does not track usage.
"""
class CountableResource(BaseResource):
"""Describe a resource where the counts are determined by a function."""
def __init__(self, name, count, flag=None):
"""Initializes a CountableResource.
Countable resources are those resources which directly
correspond to objects in the database, i.e., netowk, subnet,
etc.,. A CountableResource must be constructed with a counting
function, which will be called to determine the current counts
of the resource.
The counting function will be passed the context, along with
the extra positional and keyword arguments that are passed to
Quota.count(). It should return an integer specifying the
count.
:param name: The name of the resource, i.e., "instances".
:param count: A callable which returns the count of the
resource. The arguments passed are as described
above.
:param flag: The name of the flag or configuration option
which specifies the default value of the quota
for this resource.
"""
super(CountableResource, self).__init__(name, flag=flag)
self.count = count
class TrackedResource(BaseResource):
"""Resource which keeps track of its usage data."""
def __init__(self, name, model_class, flag):
"""Initializes an instance for a given resource.
TrackedResource are directly mapped to data model classes.
Resource usage is tracked in the database, and the model class to
which this resource refers is monitored to ensure always "fresh"
usage data are employed when performing quota checks.
This class operates under the assumption that the model class
describing the resource has a tenant identifier attribute.
:param name: The name of the resource, i.e., "networks".
:param model_class: The sqlalchemy model class of the resource for
which this instance is being created
:param flag: The name of the flag or configuration option
which specifies the default value of the quota
for this resource.
"""
super(TrackedResource, self).__init__(name, flag)
# Register events for addition/removal of records in the model class
# As tenant_id is immutable for all Neutron objects there is no need
# to register a listener for update events
self._model_class = model_class
self._dirty_tenants = set()
self._out_of_sync_tenants = set()
@property
def dirty(self):
return self._dirty_tenants
@lockutils.synchronized('dirty_tenants')
def mark_dirty(self, context, nested=False):
if not self._dirty_tenants:
return
with context.session.begin(nested=nested, subtransactions=True):
for tenant_id in self._dirty_tenants:
quota_api.set_quota_usage_dirty(context, self.name, tenant_id)
LOG.debug(("Persisted dirty status for tenant:%(tenant_id)s "
"on resource:%(resource)s"),
{'tenant_id': tenant_id, 'resource': self.name})
self._out_of_sync_tenants |= self._dirty_tenants
self._dirty_tenants.clear()
@lockutils.synchronized('dirty_tenants')
def _db_event_handler(self, mapper, _conn, target):
tenant_id = target.get('tenant_id')
if not tenant_id:
# NOTE: This is an unexpected error condition. Log anomaly but do
# not raise as this might have unexpected effects on other
# operations
LOG.error(_LE("Model class %s does not have tenant_id attribute"),
target)
return
self._dirty_tenants.add(tenant_id)
# Retry the operation if a duplicate entry exception is raised. This
# can happen is two or more workers are trying to create a resource of a
# give kind for the same tenant concurrently. Retrying the operation will
# ensure that an UPDATE statement is emitted rather than an INSERT one
@oslo_db_api.wrap_db_retry(
max_retries=db_api.MAX_RETRIES,
exception_checker=lambda exc:
isinstance(exc, oslo_db_exception.DBDuplicateEntry))
def _set_quota_usage(self, context, tenant_id, in_use):
return quota_api.set_quota_usage(context, self.name,
tenant_id, in_use=in_use)
def _resync(self, context, tenant_id, in_use):
# Update quota usage
usage_info = self._set_quota_usage(
context, tenant_id, in_use=in_use)
self._dirty_tenants.discard(tenant_id)
self._out_of_sync_tenants.discard(tenant_id)
LOG.debug(("Unset dirty status for tenant:%(tenant_id)s on "
"resource:%(resource)s"),
{'tenant_id': tenant_id, 'resource': self.name})
return usage_info
def resync(self, context, tenant_id):
if tenant_id not in self._out_of_sync_tenants:
return
LOG.debug(("Synchronizing usage tracker for tenant:%(tenant_id)s on "
"resource:%(resource)s"),
{'tenant_id': tenant_id, 'resource': self.name})
in_use = context.session.query(self._model_class).filter_by(
tenant_id=tenant_id).count()
# Update quota usage
return self._resync(context, tenant_id, in_use)
def count(self, context, _plugin, _resources, tenant_id,
resync_usage=False):
"""Return the current usage count for the resource."""
# Load current usage data
usage_info = quota_api.get_quota_usage_by_resource_and_tenant(
context, self.name, tenant_id)
# If dirty or missing, calculate actual resource usage querying
# the database and set/create usage info data
# NOTE: this routine "trusts" usage counters at service startup. This
# assumption is generally valid, but if the database is tampered with,
# or if data migrations do not take care of usage counters, the
# assumption will not hold anymore
if (tenant_id in self._dirty_tenants or not usage_info
or usage_info.dirty):
LOG.debug(("Usage tracker for resource:%(resource)s and tenant:"
"%(tenant_id)s is out of sync, need to count used "
"quota"), {'resource': self.name,
'tenant_id': tenant_id})
in_use = context.session.query(self._model_class).filter_by(
tenant_id=tenant_id).count()
# Update quota usage, if requested (by default do not do that, as
# typically one counts before adding a record, and that would mark
# the usage counter as dirty again)
if resync_usage or not usage_info:
usage_info = self._resync(context, tenant_id, in_use)
else:
usage_info = quota_api.QuotaUsageInfo(usage_info.resource,
usage_info.tenant_id,
in_use,
usage_info.reserved,
usage_info.dirty)
return usage_info.total
def register_events(self):
event.listen(self._model_class, 'after_insert', self._db_event_handler)
event.listen(self._model_class, 'after_delete', self._db_event_handler)
def unregister_events(self):
event.remove(self._model_class, 'after_insert', self._db_event_handler)
event.remove(self._model_class, 'after_delete', self._db_event_handler)

245
neutron/quota/resource_registry.py Normal file
View File

@ -0,0 +1,245 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from oslo_log import log
import six
from neutron.i18n import _LI, _LW
from neutron.quota import resource
LOG = log.getLogger(__name__)
# Wrappers for easing access to the ResourceRegistry singleton
def register_resource(resource):
ResourceRegistry.get_instance().register_resource(resource)
def register_resource_by_name(resource_name):
ResourceRegistry.get_instance().register_resource_by_name(resource_name)
def get_all_resources():
return ResourceRegistry.get_instance().resources
def get_resource(resource_name):
return ResourceRegistry.get_instance().get_resource(resource_name)
def is_tracked(resource_name):
return ResourceRegistry.get_instance().is_tracked(resource_name)
# auxiliary functions and decorators
def set_resources_dirty(context):
"""Sets the dirty bit for resources with usage changes.
This routine scans all registered resources, and, for those whose
dirty status is True, sets the dirty bit to True in the database
for the appropriate tenants.
Please note that this routine begins a nested transaction, and it
is not recommended that this transaction begins within another
transaction. For this reason the function will raise a SqlAlchemy
exception if such an attempt is made.
:param context: a Neutron request context with a DB session
"""
if not cfg.CONF.QUOTAS.track_quota_usage:
return
for res in get_all_resources().values():
with context.session.begin():
if is_tracked(res.name) and res.dirty:
res.mark_dirty(context, nested=True)
def resync_resource(context, resource_name, tenant_id):
if not cfg.CONF.QUOTAS.track_quota_usage:
return
if is_tracked(resource_name):
res = get_resource(resource_name)
# If the resource is tracked count supports the resync_usage parameter
res.resync(context, tenant_id)
def mark_resources_dirty(f):
"""Decorator for functions which alter resource usage.
This decorator ensures set_resource_dirty is invoked after completion
of the decorated function.
"""
@six.wraps(f)
def wrapper(_self, context, *args, **kwargs):
ret_val = f(_self, context, *args, **kwargs)
set_resources_dirty(context)
return ret_val
return wrapper
class tracked_resources(object):
"""Decorator for specifying resources for which usage should be tracked.
A plugin class can use this decorator to specify for which resources
usage info should be tracked into an appropriate table rather than being
explicitly counted.
"""
def __init__(self, override=False, **kwargs):
self._tracked_resources = kwargs
self._override = override
def __call__(self, f):
@six.wraps(f)
def wrapper(*args, **kwargs):
registry = ResourceRegistry.get_instance()
for resource_name in self._tracked_resources:
registry.set_tracked_resource(
resource_name,
self._tracked_resources[resource_name],
self._override)
return f(*args, **kwargs)
return wrapper
class ResourceRegistry(object):
"""Registry for resource subject to quota limits.
This class keeps track of Neutron resources for which quota limits are
enforced, regardless of whether their usage is being tracked or counted.
For tracked-usage resources, that is to say those resources for which
there are usage counters which are kept in sync with the actual number
of rows in the database, this class allows the plugin to register their
names either explicitly or through the @tracked_resources decorator,
which should preferrably be applied to the __init__ method of the class.
"""
_instance = None
@classmethod
def get_instance(cls):
if cls._instance is None:
cls._instance = cls()
return cls._instance
def __init__(self):
self._resources = {}
# Map usage tracked resources to the correspondent db model class
self._tracked_resource_mappings = {}
def __contains__(self, resource):
return resource in self._resources
def _create_resource_instance(self, resource_name):
"""Factory function for quota Resource.
This routine returns a resource instance of the appropriate type
according to system configuration.
If QUOTAS.track_quota_usage is True, and there is a model mapping for
the current resource, this function will return an instance of
AccountedResource; otherwise an instance of CountableResource.
"""
if (not cfg.CONF.QUOTAS.track_quota_usage or
resource_name not in self._tracked_resource_mappings):
LOG.info(_LI("Creating instance of CountableResource for "
"resource:%s"), resource_name)
return resource.CountableResource(
resource_name, resource._count_resource,
'quota_%s' % resource_name)
else:
LOG.info(_LI("Creating instance of TrackedResource for "
"resource:%s"), resource_name)
return resource.TrackedResource(
resource_name,
self._tracked_resource_mappings[resource_name],
'quota_%s' % resource_name)
def set_tracked_resource(self, resource_name, model_class, override=False):
# Do not do anything if tracking is disabled by config
if not cfg.CONF.QUOTAS.track_quota_usage:
return
current_model_class = self._tracked_resource_mappings.setdefault(
resource_name, model_class)
# Check whether setdefault also set the entry in the dict
if current_model_class != model_class:
LOG.debug("A model class is already defined for %(resource)s: "
"%(current_model_class)s. Override:%(override)s",
{'resource': resource_name,
'current_model_class': current_model_class,
'override': override})
if override:
self._tracked_resource_mappings[resource_name] = model_class
LOG.debug("Tracking information for resource: %s configured",
resource_name)
def is_tracked(self, resource_name):
"""Find out if a resource if tracked or not.
:param resource_name: name of the resource.
:returns True if resource_name is registered and tracked, otherwise
False. Please note that here when False it returned it
simply means that resource_name is not a TrackedResource
instance, it does not necessarily mean that the resource
is not registered.
"""
return resource_name in self._tracked_resource_mappings
def register_resource(self, resource):
if resource.name in self._resources:
LOG.warn(_LW('%s is already registered'), resource.name)
if resource.name in self._tracked_resource_mappings:
resource.register_events()
self._resources[resource.name] = resource
def register_resources(self, resources):
for res in resources:
self.register_resource(res)
def register_resource_by_name(self, resource_name):
"""Register a resource by name."""
resource = self._create_resource_instance(resource_name)
self.register_resource(resource)
def unregister_resources(self):
"""Unregister all resources."""
for (res_name, res) in self._resources.items():
if res_name in self._tracked_resource_mappings:
res.unregister_events()
self._resources.clear()
self._tracked_resource_mappings.clear()
def get_resource(self, resource_name):
"""Return a resource given its name.
:returns: The resource instance or None if the resource is not found
"""
return self._resources.get(resource_name)
@property
def resources(self):
return self._resources

27
neutron/tests/unit/api/v2/test_base.py
View File

@ -37,6 +37,7 @@ from neutron import context
from neutron import manager
from neutron import policy
from neutron import quota
from neutron.quota import resource_registry
from neutron.tests import base
from neutron.tests import fake_notifier
from neutron.tests import tools
@ -1289,6 +1290,12 @@ class NotificationTest(APIv2TestBase):
class DHCPNotificationTest(APIv2TestBase):
def setUp(self):
# This test does not have database support so tracking cannot be used
cfg.CONF.set_override('track_quota_usage', False, group='QUOTAS')
super(DHCPNotificationTest, self).setUp()
def _test_dhcp_notifier(self, opname, resource, initial_input=None):
instance = self.plugin.return_value
instance.get_networks.return_value = initial_input
@ -1340,6 +1347,23 @@ class DHCPNotificationTest(APIv2TestBase):
class QuotaTest(APIv2TestBase):
def setUp(self):
# This test does not have database support so tracking cannot be used
cfg.CONF.set_override('track_quota_usage', False, group='QUOTAS')
super(QuotaTest, self).setUp()
# Use mock to let the API use a different QuotaEngine instance for
# unit test in this class. This will ensure resource are registered
# again and instanciated with neutron.quota.resource.CountableResource
replacement_registry = resource_registry.ResourceRegistry()
registry_patcher = mock.patch('neutron.quota.resource_registry.'
'ResourceRegistry.get_instance')
mock_registry = registry_patcher.start().return_value
mock_registry.get_resource = replacement_registry.get_resource
mock_registry.resources = replacement_registry.resources
# Register a resource
replacement_registry.register_resource_by_name('network')
def test_create_network_quota(self):
cfg.CONF.set_override('quota_network', 1, group='QUOTAS')
initial_input = {'network': {'name': 'net1', 'tenant_id': _uuid()}}
@ -1384,9 +1408,10 @@ class QuotaTest(APIv2TestBase):
class ExtensionTestCase(base.BaseTestCase):
def setUp(self):
# This test does not have database support so tracking cannot be used
cfg.CONF.set_override('track_quota_usage', False, group='QUOTAS')
super(ExtensionTestCase, self).setUp()
plugin = 'neutron.neutron_plugin_base_v2.NeutronPluginBaseV2'
# Ensure existing ExtensionManager is not used
extensions.PluginAwareExtensionManager._instance = None

4
neutron/tests/unit/extensions/extensionattribute.py
View File

@ -18,7 +18,7 @@ import abc
from neutron.api import extensions
from neutron.api.v2 import base
from neutron import manager
from neutron import quota
from neutron.quota import resource_registry
# Attribute Map
@ -69,7 +69,7 @@ class Extensionattribute(extensions.ExtensionDescriptor):
collection_name = resource_name + "s"
params = RESOURCE_ATTRIBUTE_MAP.get(collection_name, dict())
quota.QUOTAS.register_resource_by_name(resource_name)
resource_registry.register_resource_by_name(resource_name)
controller = base.create_resource(collection_name,
resource_name,

3
neutron/tests/unit/extensions/test_quotasv2.py
View File

@ -29,6 +29,7 @@ from neutron.common import exceptions
from neutron import context
from neutron.db.quota import driver
from neutron import quota
from neutron.quota import resource_registry
from neutron.tests import base
from neutron.tests import tools
from neutron.tests.unit.api.v2 import test_base
@ -64,7 +65,7 @@ class QuotaExtensionTestCase(testlib_api.WebTestCase):
self.plugin.return_value.supported_extension_aliases = ['quotas']
# QUOTAS will register the items in conf when starting
# extra1 here is added later, so have to do it manually
quota.QUOTAS.register_resource_by_name('extra1')
resource_registry.register_resource_by_name('extra1')
ext_mgr = extensions.PluginAwareExtensionManager.get_instance()
app = config.load_paste_app('extensions_test_app')
ext_middleware = extensions.ExtensionMiddleware(app, ext_mgr=ext_mgr)

28
neutron/tests/unit/quota/__init__.py Normal file
View File

@ -0,0 +1,28 @@
# Copyright (c) 2015 OpenStack Foundation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sa
from neutron.db import model_base
from neutron.db import models_v2
# Model classes for test resources
class MehModel(model_base.BASEV2, models_v2.HasTenant):
meh = sa.Column(sa.String(8), primary_key=True)
class OtherMehModel(model_base.BASEV2, models_v2.HasTenant):
othermeh = sa.Column(sa.String(8), primary_key=True)

232
neutron/tests/unit/quota/test_resource.py Normal file
View File

@ -0,0 +1,232 @@
# Copyright (c) 2015 OpenStack Foundation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import random
import mock
from oslo_config import cfg
from neutron import context
from neutron.db import api as db_api
from neutron.db.quota import api as quota_api
from neutron.quota import resource
from neutron.tests import base
from neutron.tests.unit import quota as test_quota
from neutron.tests.unit import testlib_api
meh_quota_flag = 'quota_meh'
meh_quota_opts = [cfg.IntOpt(meh_quota_flag, default=99)]
random.seed()
class TestTrackedResource(testlib_api.SqlTestCaseLight):
def _add_data(self, tenant_id=None):
session = db_api.get_session()
with session.begin():
tenant_id = tenant_id or self.tenant_id
session.add(test_quota.MehModel(
meh='meh_%d' % random.randint(0, 10000),
tenant_id=tenant_id))
session.add(test_quota.MehModel(
meh='meh_%d' % random.randint(0, 10000),
tenant_id=tenant_id))
def _delete_data(self):
session = db_api.get_session()
with session.begin():
query = session.query(test_quota.MehModel).filter_by(
tenant_id=self.tenant_id)
for item in query:
session.delete(item)
def _update_data(self):
session = db_api.get_session()
with session.begin():
query = session.query(test_quota.MehModel).filter_by(
tenant_id=self.tenant_id)
for item in query:
item['meh'] = 'meh-%s' % item['meh']
session.add(item)
def setUp(self):
base.BaseTestCase.config_parse()
cfg.CONF.register_opts(meh_quota_opts, 'QUOTAS')
self.addCleanup(cfg.CONF.reset)
self.resource = 'meh'
self.other_resource = 'othermeh'
self.tenant_id = 'meh'
self.context = context.Context(
user_id='', tenant_id=self.tenant_id, is_admin=False)
super(TestTrackedResource, self).setUp()
def _register_events(self, res):
res.register_events()
self.addCleanup(res.unregister_events)
def _create_resource(self):
res = resource.TrackedResource(
self.resource, test_quota.MehModel, meh_quota_flag)
self._register_events(res)
return res
def _create_other_resource(self):
res = resource.TrackedResource(
self.other_resource, test_quota.OtherMehModel, meh_quota_flag)
self._register_events(res)
return res
def test_count_first_call_with_dirty_false(self):
quota_api.set_quota_usage(
self.context, self.resource, self.tenant_id, in_use=1)
res = self._create_resource()
self._add_data()
# explicitly set dirty flag to False
quota_api.set_all_quota_usage_dirty(
self.context, self.resource, dirty=False)
# Expect correct count to be returned anyway since the first call to
# count() always resyncs with the db
self.assertEqual(2, res.count(self.context,
None, None,
self.tenant_id))
def _test_count(self):
res = self._create_resource()
quota_api.set_quota_usage(
self.context, res.name, self.tenant_id, in_use=0)
self._add_data()
return res
def test_count_with_dirty_false(self):
res = self._test_count()
res.count(self.context, None, None, self.tenant_id)
# At this stage count has been invoked, and the dirty flag should be
# false. Another invocation of count should not query the model class
set_quota = 'neutron.db.quota.api.set_quota_usage'
with mock.patch(set_quota) as mock_set_quota:
self.assertEqual(0, mock_set_quota.call_count)
self.assertEqual(2, res.count(self.context,
None, None,
self.tenant_id))
def test_count_with_dirty_true_resync(self):
res = self._test_count()
# Expect correct count to be returned, which also implies
# set_quota_usage has been invoked with the correct parameters
self.assertEqual(2, res.count(self.context,
None, None,
self.tenant_id,
resync_usage=True))
def test_count_with_dirty_true_resync_calls_set_quota_usage(self):
res = self._test_count()
set_quota_usage = 'neutron.db.quota.api.set_quota_usage'
with mock.patch(set_quota_usage) as mock_set_quota_usage:
quota_api.set_quota_usage_dirty(self.context,
self.resource,
self.tenant_id)
res.count(self.context, None, None, self.tenant_id,
resync_usage=True)
mock_set_quota_usage.assert_called_once_with(
self.context, self.resource, self.tenant_id, in_use=2)
def test_count_with_dirty_true_no_usage_info(self):
res = self._create_resource()
self._add_data()
# Invoke count without having usage info in DB - Expect correct
# count to be returned
self.assertEqual(2, res.count(self.context,
None, None,
self.tenant_id))
def test_count_with_dirty_true_no_usage_info_calls_set_quota_usage(self):
res = self._create_resource()
self._add_data()
set_quota_usage = 'neutron.db.quota.api.set_quota_usage'
with mock.patch(set_quota_usage) as mock_set_quota_usage:
quota_api.set_quota_usage_dirty(self.context,
self.resource,
self.tenant_id)
res.count(self.context, None, None, self.tenant_id,
resync_usage=True)
mock_set_quota_usage.assert_called_once_with(
self.context, self.resource, self.tenant_id, in_use=2)
def test_add_delete_data_triggers_event(self):
res = self._create_resource()
other_res = self._create_other_resource()
# Validate dirty tenants since mock does not work well with sqlalchemy
# event handlers.
self._add_data()
self._add_data('someone_else')
self.assertEqual(2, len(res._dirty_tenants))
# Also, the dirty flag should not be set for other resources
self.assertEqual(0, len(other_res._dirty_tenants))
self.assertIn(self.tenant_id, res._dirty_tenants)
self.assertIn('someone_else', res._dirty_tenants)
def test_delete_data_triggers_event(self):
res = self._create_resource()
self._add_data()
self._add_data('someone_else')
# Artificially clear _dirty_tenants
res._dirty_tenants.clear()
self._delete_data()
# We did not delete "someone_else", so expect only a single dirty
# tenant
self.assertEqual(1, len(res._dirty_tenants))
self.assertIn(self.tenant_id, res._dirty_tenants)
def test_update_does_not_trigger_event(self):
res = self._create_resource()
self._add_data()
self._add_data('someone_else')
# Artificially clear _dirty_tenants
res._dirty_tenants.clear()
self._update_data()
self.assertEqual(0, len(res._dirty_tenants))
def test_mark_dirty(self):
res = self._create_resource()
self._add_data()
self._add_data('someone_else')
set_quota_usage = 'neutron.db.quota.api.set_quota_usage_dirty'
with mock.patch(set_quota_usage) as mock_set_quota_usage:
res.mark_dirty(self.context)
self.assertEqual(2, mock_set_quota_usage.call_count)
mock_set_quota_usage.assert_any_call(
self.context, self.resource, self.tenant_id)
mock_set_quota_usage.assert_any_call(
self.context, self.resource, 'someone_else')
def test_mark_dirty_no_dirty_tenant(self):
res = self._create_resource()
set_quota_usage = 'neutron.db.quota.api.set_quota_usage_dirty'
with mock.patch(set_quota_usage) as mock_set_quota_usage:
res.mark_dirty(self.context)
self.assertFalse(mock_set_quota_usage.call_count)
def test_resync(self):
res = self._create_resource()
self._add_data()
res.mark_dirty(self.context)
# self.tenant_id now is out of sync
set_quota_usage = 'neutron.db.quota.api.set_quota_usage'
with mock.patch(set_quota_usage) as mock_set_quota_usage:
res.resync(self.context, self.tenant_id)
# and now it should be in sync
self.assertNotIn(self.tenant_id, res._out_of_sync_tenants)
mock_set_quota_usage.assert_called_once_with(
self.context, self.resource, self.tenant_id, in_use=2)

159
neutron/tests/unit/quota/test_resource_registry.py Normal file
View File

@ -0,0 +1,159 @@
# Copyright (c) 2015 OpenStack Foundation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from oslo_config import cfg
from neutron import context
from neutron.quota import resource
from neutron.quota import resource_registry
from neutron.tests import base
from neutron.tests.unit import quota as test_quota
class TestResourceRegistry(base.DietTestCase):
def setUp(self):
super(TestResourceRegistry, self).setUp()
self.registry = resource_registry.ResourceRegistry.get_instance()
# clean up the registry at every test
self.registry.unregister_resources()
def test_set_tracked_resource_new_resource(self):
self.registry.set_tracked_resource('meh', test_quota.MehModel)
self.assertEqual(test_quota.MehModel,
self.registry._tracked_resource_mappings['meh'])
def test_set_tracked_resource_existing_with_override(self):
self.test_set_tracked_resource_new_resource()
self.registry.set_tracked_resource('meh', test_quota.OtherMehModel,
override=True)
# Overidde is set to True, the model class should change
self.assertEqual(test_quota.OtherMehModel,
self.registry._tracked_resource_mappings['meh'])
def test_set_tracked_resource_existing_no_override(self):
self.test_set_tracked_resource_new_resource()
self.registry.set_tracked_resource('meh', test_quota.OtherMehModel)
# Overidde is set to false, the model class should not change
self.assertEqual(test_quota.MehModel,
self.registry._tracked_resource_mappings['meh'])
def _test_register_resource_by_name(self, resource_name, expected_type):
self.assertNotIn(resource_name, self.registry._resources)
self.registry.register_resource_by_name(resource_name)
self.assertIn(resource_name, self.registry._resources)
self.assertIsInstance(self.registry.get_resource(resource_name),
expected_type)
def test_register_resource_by_name_tracked(self):
self.test_set_tracked_resource_new_resource()
self._test_register_resource_by_name('meh', resource.TrackedResource)
def test_register_resource_by_name_not_tracked(self):
self._test_register_resource_by_name('meh', resource.CountableResource)
def test_register_resource_by_name_with_tracking_disabled_by_config(self):
cfg.CONF.set_override('track_quota_usage', False,
group='QUOTAS')
# DietTestCase does not automatically cleans configuration overrides
self.addCleanup(cfg.CONF.reset)
self.registry.set_tracked_resource('meh', test_quota.MehModel)
self.assertNotIn(
'meh', self.registry._tracked_resource_mappings)
self._test_register_resource_by_name('meh', resource.CountableResource)
class TestAuxiliaryFunctions(base.DietTestCase):
def setUp(self):
super(TestAuxiliaryFunctions, self).setUp()
self.registry = resource_registry.ResourceRegistry.get_instance()
# clean up the registry at every test
self.registry.unregister_resources()
def test_resync_tracking_disabled(self):
cfg.CONF.set_override('track_quota_usage', False,
group='QUOTAS')
# DietTestCase does not automatically cleans configuration overrides
self.addCleanup(cfg.CONF.reset)
with mock.patch('neutron.quota.resource.'
'TrackedResource.resync') as mock_resync:
self.registry.set_tracked_resource('meh', test_quota.MehModel)
self.registry.register_resource_by_name('meh')
resource_registry.resync_resource(mock.ANY, 'meh', 'tenant_id')
self.assertEqual(0, mock_resync.call_count)
def test_resync_tracked_resource(self):
with mock.patch('neutron.quota.resource.'
'TrackedResource.resync') as mock_resync:
self.registry.set_tracked_resource('meh', test_quota.MehModel)
self.registry.register_resource_by_name('meh')
resource_registry.resync_resource(mock.ANY, 'meh', 'tenant_id')
mock_resync.assert_called_once_with(mock.ANY, 'tenant_id')
def test_resync_non_tracked_resource(self):
with mock.patch('neutron.quota.resource.'
'TrackedResource.resync') as mock_resync:
self.registry.register_resource_by_name('meh')
resource_registry.resync_resource(mock.ANY, 'meh', 'tenant_id')
self.assertEqual(0, mock_resync.call_count)
def test_set_resources_dirty_invoked_with_tracking_disabled(self):
cfg.CONF.set_override('track_quota_usage', False,
group='QUOTAS')
# DietTestCase does not automatically cleans configuration overrides
self.addCleanup(cfg.CONF.reset)
with mock.patch('neutron.quota.resource.'
'TrackedResource.mark_dirty') as mock_mark_dirty:
self.registry.set_tracked_resource('meh', test_quota.MehModel)
self.registry.register_resource_by_name('meh')
resource_registry.set_resources_dirty(mock.ANY)
self.assertEqual(0, mock_mark_dirty.call_count)
def test_set_resources_dirty_no_dirty_resource(self):
ctx = context.Context('user_id', 'tenant_id',
is_admin=False, is_advsvc=False)
with mock.patch('neutron.quota.resource.'
'TrackedResource.mark_dirty') as mock_mark_dirty:
self.registry.set_tracked_resource('meh', test_quota.MehModel)
self.registry.register_resource_by_name('meh')
res = self.registry.get_resource('meh')
# This ensures dirty is false
res._dirty_tenants.clear()
resource_registry.set_resources_dirty(ctx)
self.assertEqual(0, mock_mark_dirty.call_count)
def test_set_resources_dirty_no_tracked_resource(self):
ctx = context.Context('user_id', 'tenant_id',
is_admin=False, is_advsvc=False)
with mock.patch('neutron.quota.resource.'
'TrackedResource.mark_dirty') as mock_mark_dirty:
self.registry.register_resource_by_name('meh')
resource_registry.set_resources_dirty(ctx)
self.assertEqual(0, mock_mark_dirty.call_count)
def test_set_resources_dirty(self):
ctx = context.Context('user_id', 'tenant_id',
is_admin=False, is_advsvc=False)
with mock.patch('neutron.quota.resource.'
'TrackedResource.mark_dirty') as mock_mark_dirty:
self.registry.set_tracked_resource('meh', test_quota.MehModel)
self.registry.register_resource_by_name('meh')
res = self.registry.get_resource('meh')
# This ensures dirty is true
res._dirty_tenants.add('tenant_id')
resource_registry.set_resources_dirty(ctx)
mock_mark_dirty.assert_called_once_with(ctx, nested=True)