From c8daf8b1d893ea4bfa8d44b030b7a07926e0c3ba Mon Sep 17 00:00:00 2001 From: melanie witt Date: Fri, 13 Oct 2017 01:45:13 +0000 Subject: [PATCH] Remove old-style quotas code In Pike, we re-architected quotas to count resources for quota usage instead of tracking it separately and stopped using reservations. During an upgrade to Queens, old computes (Pike) will be using new-style quotas and so the old-style quotas code is no longer needed. Change-Id: Ie01ab1c3a1219f1d123f0ecedc66a00dfb2eb2c1 --- nova/db/api.py | 68 ---- nova/db/sqlalchemy/api.py | 520 ------------------------ nova/objects/quotas.py | 49 +-- nova/quota.py | 541 +------------------------ nova/tests/unit/db/test_db_api.py | 461 +-------------------- nova/tests/unit/objects/test_quotas.py | 87 ---- nova/tests/unit/test_quota.py | 109 +---- 7 files changed, 21 insertions(+), 1814 deletions(-) diff --git a/nova/db/api.py b/nova/db/api.py index 9e5c9a649ac7..d21c0171c704 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -1198,74 +1198,6 @@ def quota_class_update(context, class_name, resource, limit): ################### -def quota_usage_get(context, project_id, resource, user_id=None): - """Retrieve a quota usage or raise if it does not exist.""" - return IMPL.quota_usage_get(context, project_id, resource, user_id=user_id) - - -def quota_usage_get_all_by_project_and_user(context, project_id, user_id): - """Retrieve all usage associated with a given resource.""" - return IMPL.quota_usage_get_all_by_project_and_user(context, - project_id, user_id) - - -def quota_usage_get_all_by_project(context, project_id): - """Retrieve all usage associated with a given resource.""" - return IMPL.quota_usage_get_all_by_project(context, project_id) - - -def quota_usage_update(context, project_id, user_id, resource, **kwargs): - """Update a quota usage or raise if it does not exist.""" - return IMPL.quota_usage_update(context, project_id, user_id, resource, - **kwargs) - - -def quota_usage_refresh(context, resources, keys, until_refresh, max_age, - project_id=None, user_id=None): - """Refresh the quota usages. - - :param context: The request context, for access checks. - :param resources: A dictionary of the registered resources. - :param keys: Names of the resources whose usage is to be refreshed. - :param until_refresh: The until_refresh configuration value. - :param max_age: The max_age configuration value. - :param project_id: (Optional) The project_id containing the usages - to be refreshed. Defaults to the project_id - in the context. - :param user_id: (Optional) The user_id containing the usages - to be refreshed. Defaults to the user_id - in the context. - - """ - return IMPL.quota_usage_refresh(context, resources, keys, until_refresh, - max_age, project_id=project_id, user_id=user_id) - - -################### - - -def quota_reserve(context, resources, quotas, user_quotas, deltas, expire, - until_refresh, max_age, project_id=None, user_id=None): - """Check quotas and create appropriate reservations.""" - return IMPL.quota_reserve(context, resources, quotas, user_quotas, deltas, - expire, until_refresh, max_age, - project_id=project_id, user_id=user_id) - - -def reservation_commit(context, reservations, project_id=None, user_id=None): - """Commit quota reservations.""" - return IMPL.reservation_commit(context, reservations, - project_id=project_id, - user_id=user_id) - - -def reservation_rollback(context, reservations, project_id=None, user_id=None): - """Roll back quota reservations.""" - return IMPL.reservation_rollback(context, reservations, - project_id=project_id, - user_id=user_id) - - def quota_destroy_all_by_project_and_user(context, project_id, user_id): """Destroy all quotas associated with a given project and user.""" return IMPL.quota_destroy_all_by_project_and_user(context, diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index c2bfa7dc914c..05341b40001e 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -352,37 +352,6 @@ def convert_objects_related_datetimes(values, *datetime_keys): return values -def _sync_instances(context, project_id, user_id): - return dict(zip(('instances', 'cores', 'ram'), - _instance_data_get_for_user(context, project_id, user_id))) - - -def _sync_floating_ips(context, project_id, user_id): - return dict(floating_ips=_floating_ip_count_by_project( - context, project_id)) - - -def _sync_fixed_ips(context, project_id, user_id): - return dict(fixed_ips=_fixed_ip_count_by_project(context, project_id)) - - -def _sync_security_groups(context, project_id, user_id): - return dict(security_groups=_security_group_count_by_project_and_user( - context, project_id, user_id)) - - -def _sync_server_groups(context, project_id, user_id): - return dict(server_groups=_instance_group_count_by_project_and_user( - context, project_id, user_id)) - -QUOTA_SYNC_FUNCTIONS = { - '_sync_instances': _sync_instances, - '_sync_floating_ips': _sync_floating_ips, - '_sync_fixed_ips': _sync_fixed_ips, - '_sync_security_groups': _sync_security_groups, - '_sync_server_groups': _sync_server_groups, -} - ################### @@ -3595,481 +3564,6 @@ def quota_class_update(context, class_name, resource, limit): ################### -@require_context -@pick_context_manager_reader -def quota_usage_get(context, project_id, resource, user_id=None): - query = model_query(context, models.QuotaUsage, read_deleted="no").\ - filter_by(project_id=project_id).\ - filter_by(resource=resource) - if user_id: - if resource not in PER_PROJECT_QUOTAS: - result = query.filter_by(user_id=user_id).first() - else: - result = query.filter_by(user_id=None).first() - else: - result = query.first() - - if not result: - raise exception.QuotaUsageNotFound(project_id=project_id) - - return result - - -def _quota_usage_get_all(context, project_id, user_id=None): - query = model_query(context, models.QuotaUsage, read_deleted="no").\ - filter_by(project_id=project_id) - result = {'project_id': project_id} - if user_id: - query = query.filter(or_(models.QuotaUsage.user_id == user_id, - models.QuotaUsage.user_id == null())) - result['user_id'] = user_id - - rows = query.all() - for row in rows: - if row.resource in result: - result[row.resource]['in_use'] += row.in_use - result[row.resource]['reserved'] += row.reserved - else: - result[row.resource] = dict(in_use=row.in_use, - reserved=row.reserved) - - return result - - -@require_context -@pick_context_manager_reader -def quota_usage_get_all_by_project_and_user(context, project_id, user_id): - return _quota_usage_get_all(context, project_id, user_id=user_id) - - -@require_context -@pick_context_manager_reader -def quota_usage_get_all_by_project(context, project_id): - return _quota_usage_get_all(context, project_id) - - -def _quota_usage_create(project_id, user_id, resource, in_use, - reserved, until_refresh, session): - quota_usage_ref = models.QuotaUsage() - quota_usage_ref.project_id = project_id - quota_usage_ref.user_id = user_id - quota_usage_ref.resource = resource - quota_usage_ref.in_use = in_use - quota_usage_ref.reserved = reserved - quota_usage_ref.until_refresh = until_refresh - # updated_at is needed for judgement of max_age - quota_usage_ref.updated_at = timeutils.utcnow() - - quota_usage_ref.save(session) - - return quota_usage_ref - - -@pick_context_manager_writer -def quota_usage_update(context, project_id, user_id, resource, **kwargs): - updates = {} - - for key in ['in_use', 'reserved', 'until_refresh']: - if key in kwargs: - updates[key] = kwargs[key] - - result = model_query(context, models.QuotaUsage, read_deleted="no").\ - filter_by(project_id=project_id).\ - filter_by(resource=resource).\ - filter(or_(models.QuotaUsage.user_id == user_id, - models.QuotaUsage.user_id == null())).\ - update(updates) - - if not result: - raise exception.QuotaUsageNotFound(project_id=project_id) - - -################### - - -def _reservation_create(uuid, usage, project_id, user_id, resource, - delta, expire, session): - reservation_ref = models.Reservation() - reservation_ref.uuid = uuid - reservation_ref.usage_id = usage['id'] - reservation_ref.project_id = project_id - reservation_ref.user_id = user_id - reservation_ref.resource = resource - reservation_ref.delta = delta - reservation_ref.expire = expire - reservation_ref.save(session) - return reservation_ref - - -################### - - -# NOTE(johannes): The quota code uses SQL locking to ensure races don't -# cause under or over counting of resources. To avoid deadlocks, this -# code always acquires the lock on quota_usages before acquiring the lock -# on reservations. - -def _get_project_user_quota_usages(context, project_id, user_id): - rows = model_query(context, models.QuotaUsage, - read_deleted="no").\ - filter_by(project_id=project_id).\ - order_by(models.QuotaUsage.id.asc()).\ - with_lockmode('update').\ - all() - proj_result = dict() - user_result = dict() - # Get the total count of in_use,reserved - for row in rows: - proj_result.setdefault(row.resource, - dict(in_use=0, reserved=0, total=0)) - proj_result[row.resource]['in_use'] += row.in_use - proj_result[row.resource]['reserved'] += row.reserved - proj_result[row.resource]['total'] += (row.in_use + row.reserved) - if row.user_id is None or row.user_id == user_id: - user_result[row.resource] = row - return proj_result, user_result - - -def _create_quota_usage_if_missing(user_usages, resource, until_refresh, - project_id, user_id, session): - """Creates a QuotaUsage record and adds to user_usages if not present. - - :param user_usages: dict of resource keys to QuotaUsage records. This is - updated if resource is not in user_usages yet or - until_refresh is not None. - :param resource: The resource being checked for quota usage. - :param until_refresh: Count of reservations until usage is refreshed, - int or None - :param project_id: The project being checked for quota usage. - :param user_id: The user being checked for quota usage. - :param session: DB session holding a transaction lock. - :return: True if a new QuotaUsage record was created and added - to user_usages, False otherwise. - """ - new_usage = None - if resource not in user_usages: - user_id_to_use = user_id - if resource in PER_PROJECT_QUOTAS: - user_id_to_use = None - new_usage = _quota_usage_create(project_id, user_id_to_use, resource, - 0, 0, until_refresh or None, session) - user_usages[resource] = new_usage - return new_usage is not None - - -def _is_quota_refresh_needed(quota_usage, max_age): - """Determines if a quota usage refresh is needed. - - :param quota_usage: A QuotaUsage object for a given resource. - :param max_age: Number of seconds between subsequent usage refreshes. - :return: True if a refresh is needed, False otherwise. - """ - refresh = False - if quota_usage.in_use < 0: - # Negative in_use count indicates a desync, so try to - # heal from that... - LOG.debug('in_use has dropped below 0; forcing refresh for ' - 'QuotaUsage: %s', dict(quota_usage)) - refresh = True - elif quota_usage.until_refresh is not None: - quota_usage.until_refresh -= 1 - if quota_usage.until_refresh <= 0: - refresh = True - elif max_age and (timeutils.utcnow() - - quota_usage.updated_at).seconds >= max_age: - refresh = True - - return refresh - - -def _refresh_quota_usages(quota_usage, until_refresh, in_use): - """Refreshes quota usage for the given resource. - - :param quota_usage: A QuotaUsage object for a given resource. - :param until_refresh: Count of reservations until usage is refreshed, - int or None - :param in_use: Actual quota usage for the resource. - """ - if quota_usage.in_use != in_use: - LOG.info('quota_usages out of sync, updating. ' - 'project_id: %(project_id)s, ' - 'user_id: %(user_id)s, ' - 'resource: %(res)s, ' - 'tracked usage: %(tracked_use)s, ' - 'actual usage: %(in_use)s', - {'project_id': quota_usage.project_id, - 'user_id': quota_usage.user_id, - 'res': quota_usage.resource, - 'tracked_use': quota_usage.in_use, - 'in_use': in_use}) - else: - LOG.debug('QuotaUsage has not changed, refresh is unnecessary for: %s', - dict(quota_usage)) - - # Update the usage - quota_usage.in_use = in_use - quota_usage.until_refresh = until_refresh or None - - -def _refresh_quota_usages_if_needed(user_usages, context, resources, keys, - project_id, user_id, until_refresh, - max_age, force_refresh=False): - elevated = context.elevated() - - # Handle usage refresh - work = set(keys) - while work: - resource = work.pop() - - # Do we need to refresh the usage? - created = _create_quota_usage_if_missing(user_usages, resource, - until_refresh, project_id, - user_id, context.session) - - refresh = force_refresh - if not refresh: - refresh = created or \ - _is_quota_refresh_needed(user_usages[resource], max_age) - - # OK, refresh the usage - if refresh: - # Grab the sync routine - sync = QUOTA_SYNC_FUNCTIONS[resources[resource].sync] - - updates = sync(elevated, project_id, user_id) - for res, in_use in updates.items(): - # Make sure we have a destination for the usage! - _create_quota_usage_if_missing(user_usages, res, - until_refresh, project_id, - user_id, context.session) - _refresh_quota_usages(user_usages[res], until_refresh, - in_use) - - # Because more than one resource may be refreshed - # by the call to the sync routine, and we don't - # want to double-sync, we make sure all refreshed - # resources are dropped from the work set. - work.discard(res) - - # NOTE(Vek): We make the assumption that the sync - # routine actually refreshes the - # resources that it is the sync routine - # for. We don't check, because this is - # a best-effort mechanism. - - -def _calculate_overquota(project_quotas, user_quotas, deltas, - project_usages, user_usages): - """Checks if any resources will go over quota based on the request. - - :param project_quotas: dict of resource quotas (limits) for the project. - :param user_quotas: dict of resource quotas (limits) for the user. - :param deltas: dict of resource keys to positive/negative quota - changes for the resources in a given operation. - :param project_usages: dict of resource keys to QuotaUsage records for the - project. - :param user_usages: dict of resource keys to QuotaUsage records for the - user. - :return: list of resources that are over-quota for the - operation. - """ - overs = [] - for res, delta in deltas.items(): - # We can't go over-quota if we're not reserving anything. - if delta >= 0: - # We can't go over-quota if we have unlimited quotas. - # over if the project usage + delta is more than project quota - if 0 <= project_quotas[res] < delta + project_usages[res]['total']: - LOG.debug('Request is over project quota for resource ' - '"%(res)s". Project limit: %(limit)s, delta: ' - '%(delta)s, current total project usage: %(total)s', - {'res': res, 'limit': project_quotas[res], - 'delta': delta, - 'total': project_usages[res]['total']}) - overs.append(res) - # We can't go over-quota if we have unlimited quotas. - # over if the user usage + delta is more than user quota - elif 0 <= user_quotas[res] < delta + user_usages[res]['total']: - LOG.debug('Request is over user quota for resource ' - '"%(res)s". User limit: %(limit)s, delta: ' - '%(delta)s, current total user usage: %(total)s', - {'res': res, 'limit': user_quotas[res], - 'delta': delta, 'total': user_usages[res]['total']}) - overs.append(res) - return overs - - -@require_context -@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) -@pick_context_manager_writer -def quota_usage_refresh(context, resources, keys, until_refresh, max_age, - project_id=None, user_id=None): - if project_id is None: - project_id = context.project_id - if user_id is None: - user_id = context.user_id - - # Get the current usages - project_usages, user_usages = _get_project_user_quota_usages( - context, project_id, user_id) - - # Force refresh of the usages - _refresh_quota_usages_if_needed(user_usages, context, resources, keys, - project_id, user_id, until_refresh, - max_age, force_refresh=True) - - -@require_context -@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) -@pick_context_manager_writer -def quota_reserve(context, resources, project_quotas, user_quotas, deltas, - expire, until_refresh, max_age, project_id=None, - user_id=None): - if project_id is None: - project_id = context.project_id - if user_id is None: - user_id = context.user_id - - # Get the current usages - project_usages, user_usages = _get_project_user_quota_usages( - context, project_id, user_id) - - _refresh_quota_usages_if_needed(user_usages, context, resources, - deltas.keys(), project_id, user_id, - until_refresh, max_age) - - # Check for deltas that would go negative - unders = [res for res, delta in deltas.items() - if delta < 0 and - delta + user_usages[res].in_use < 0] - - # Now, let's check the quotas - # NOTE(Vek): We're only concerned about positive increments. - # If a project has gone over quota, we want them to - # be able to reduce their usage without any - # problems. - for key, value in user_usages.items(): - if key not in project_usages: - LOG.debug('Copying QuotaUsage for resource "%(key)s" from ' - 'user_usages into project_usages: %(value)s', - {'key': key, 'value': dict(value)}) - project_usages[key] = value - - overs = _calculate_overquota(project_quotas, user_quotas, deltas, - project_usages, user_usages) - - # NOTE(Vek): The quota check needs to be in the transaction, - # but the transaction doesn't fail just because - # we're over quota, so the OverQuota raise is - # outside the transaction. If we did the raise - # here, our usage updates would be discarded, but - # they're not invalidated by being over-quota. - - # Create the reservations - if not overs: - reservations = [] - for res, delta in deltas.items(): - reservation = _reservation_create( - uuidutils.generate_uuid(), - user_usages[res], - project_id, - user_id, - res, delta, expire, - context.session) - reservations.append(reservation.uuid) - - # Also update the reserved quantity - # NOTE(Vek): Again, we are only concerned here about - # positive increments. Here, though, we're - # worried about the following scenario: - # - # 1) User initiates resize down. - # 2) User allocates a new instance. - # 3) Resize down fails or is reverted. - # 4) User is now over quota. - # - # To prevent this, we only update the - # reserved value if the delta is positive. - if delta > 0: - user_usages[res].reserved += delta - - # Apply updates to the usages table - for usage_ref in user_usages.values(): - context.session.add(usage_ref) - - if unders: - LOG.warning("Change will make usage less than 0 for the following " - "resources: %s", unders) - - if overs: - if project_quotas == user_quotas: - usages = project_usages - else: - # NOTE(mriedem): user_usages is a dict of resource keys to - # QuotaUsage sqlalchemy dict-like objects and doesn't log well - # so convert the user_usages values to something useful for - # logging. Remove this if we ever change how - # _get_project_user_quota_usages returns the user_usages values. - user_usages = {k: dict(in_use=v['in_use'], reserved=v['reserved'], - total=v['total']) - for k, v in user_usages.items()} - usages = user_usages - usages = {k: dict(in_use=v['in_use'], reserved=v['reserved']) - for k, v in usages.items()} - LOG.debug('Raise OverQuota exception because: ' - 'project_quotas: %(project_quotas)s, ' - 'user_quotas: %(user_quotas)s, deltas: %(deltas)s, ' - 'overs: %(overs)s, project_usages: %(project_usages)s, ' - 'user_usages: %(user_usages)s', - {'project_quotas': project_quotas, - 'user_quotas': user_quotas, - 'overs': overs, 'deltas': deltas, - 'project_usages': project_usages, - 'user_usages': user_usages}) - raise exception.OverQuota(overs=sorted(overs), quotas=user_quotas, - usages=usages) - - return reservations - - -def _quota_reservations_query(context, reservations): - """Return the relevant reservations.""" - - # Get the listed reservations - return model_query(context, models.Reservation, read_deleted="no").\ - filter(models.Reservation.uuid.in_(reservations)).\ - with_lockmode('update') - - -@require_context -@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) -@pick_context_manager_writer -def reservation_commit(context, reservations, project_id=None, user_id=None): - _project_usages, user_usages = _get_project_user_quota_usages( - context, project_id, user_id) - reservation_query = _quota_reservations_query(context, reservations) - for reservation in reservation_query.all(): - usage = user_usages[reservation.resource] - if reservation.delta >= 0: - usage.reserved -= reservation.delta - usage.in_use += reservation.delta - reservation_query.soft_delete(synchronize_session=False) - - -@require_context -@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) -@pick_context_manager_writer -def reservation_rollback(context, reservations, project_id=None, user_id=None): - _project_usages, user_usages = _get_project_user_quota_usages( - context, project_id, user_id) - reservation_query = _quota_reservations_query(context, reservations) - for reservation in reservation_query.all(): - usage = user_usages[reservation.resource] - if reservation.delta >= 0: - usage.reserved -= reservation.delta - reservation_query.soft_delete(synchronize_session=False) - - @pick_context_manager_writer def quota_destroy_all_by_project_and_user(context, project_id, user_id): model_query(context, models.ProjectUserQuota, read_deleted="no").\ @@ -4553,20 +4047,6 @@ def _security_group_ensure_default(context): 'user_id': context.user_id, 'project_id': context.project_id} default_group = security_group_create(context, values) - usage = model_query(context, models.QuotaUsage, read_deleted="no").\ - filter_by(project_id=context.project_id).\ - filter_by(user_id=context.user_id).\ - filter_by(resource='security_groups') - # Create quota usage for auto created default security group - if not usage.first(): - _quota_usage_create(context.project_id, - context.user_id, - 'security_groups', - 1, 0, - CONF.quota.until_refresh, - context.session) - else: - usage.update({'in_use': int(usage.first().in_use) + 1}) default_rules = _security_group_rule_get_default_query(context).all() for default_rule in default_rules: diff --git a/nova/objects/quotas.py b/nova/objects/quotas.py index 639db8c7d086..ebd0fb2514e8 100644 --- a/nova/objects/quotas.py +++ b/nova/objects/quotas.py @@ -63,6 +63,7 @@ class Quotas(base.NovaObject): VERSION = '1.3' fields = { + # TODO(melwitt): Remove this field in version 2.0 of the object. 'reservations': fields.ListOfStringsField(nullable=True), 'project_id': fields.StringField(nullable=True), 'user_id': fields.StringField(nullable=True), @@ -249,54 +250,27 @@ class Quotas(base.NovaObject): if not result: raise exception.QuotaClassNotFound(class_name=class_name) - @classmethod - def from_reservations(cls, context, reservations, instance=None): - """Transitional for compatibility.""" - if instance is None: - project_id = None - user_id = None - else: - project_id, user_id = ids_from_instance(context, instance) - quotas = cls() - quotas._context = context - quotas.reservations = reservations - quotas.project_id = project_id - quotas.user_id = user_id - quotas.obj_reset_changes() - return quotas - + # TODO(melwitt): Remove this method in version 2.0 of the object. @base.remotable def reserve(self, expire=None, project_id=None, user_id=None, **deltas): - reservations = quota.QUOTAS.reserve(self._context, expire=expire, - project_id=project_id, - user_id=user_id, - **deltas) - self.reservations = reservations + # Honor the expected attributes even though we're not reserving + # anything anymore. This will protect against things exploding if a + # someone has an Ocata compute host running by accident, for example. + self.reservations = None self.project_id = project_id self.user_id = user_id self.obj_reset_changes() + # TODO(melwitt): Remove this method in version 2.0 of the object. @base.remotable def commit(self): - if not self.reservations: - return - quota.QUOTAS.commit(self._context, self.reservations, - project_id=self.project_id, - user_id=self.user_id) - self.reservations = None - self.obj_reset_changes() + pass + # TODO(melwitt): Remove this method in version 2.0 of the object. @base.remotable def rollback(self): - """Rollback quotas.""" - if not self.reservations: - return - quota.QUOTAS.rollback(self._context, self.reservations, - project_id=self.project_id, - user_id=self.user_id) - self.reservations = None - self.obj_reset_changes() + pass @base.remotable_classmethod def limit_check(cls, context, project_id=None, user_id=None, **values): @@ -504,13 +478,16 @@ class Quotas(base.NovaObject): @base.NovaObjectRegistry.register class QuotasNoOp(Quotas): + # TODO(melwitt): Remove this method in version 2.0 of the object. def reserve(context, expire=None, project_id=None, user_id=None, **deltas): pass + # TODO(melwitt): Remove this method in version 2.0 of the object. def commit(self, context=None): pass + # TODO(melwitt): Remove this method in version 2.0 of the object. def rollback(self, context=None): pass diff --git a/nova/quota.py b/nova/quota.py index 0e95b15d28a4..44c354bbd748 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -17,18 +17,15 @@ """Quotas for resources per project.""" import copy -import datetime from oslo_log import log as logging from oslo_utils import importutils -from oslo_utils import timeutils import six import nova.conf from nova import context as nova_context from nova import db from nova import exception -from nova.i18n import _LE from nova import objects from nova import utils @@ -180,10 +177,8 @@ class DbQuotaDriver(object): """ usages = {} for resource in resources.values(): - # NOTE(melwitt): This is to keep things working while we're in the - # middle of converting ReservableResources to CountableResources. - # We should skip resources that are not countable and eventually - # when there are no more ReservableResources, we won't need this. + # NOTE(melwitt): We should skip resources that are not countable, + # such as AbsoluteResources. if not isinstance(resource, CountableResource): continue if resource.name in usages: @@ -373,33 +368,6 @@ class DbQuotaDriver(object): settable_quotas[key] = {'minimum': minimum, 'maximum': -1} return settable_quotas - def _get_syncable_resources(self, resources, user_id=None): - """Given a list of resources, retrieve the syncable resources - scoped to a project or a user. - - A resource is syncable if it has a function to sync the quota - usage record with the actual usage of the project or user. - - :param resources: A dictionary of the registered resources. - :param user_id: Optional. If user_id is specified, user-scoped - resources will be returned. Otherwise, - project-scoped resources will be returned. - :returns: A list of resource names scoped to a project or - user that can be sync'd. - """ - syncable_resources = [] - per_project_resources = db.quota_get_per_project_resources() - for key, value in resources.items(): - if isinstance(value, ReservableResource): - # Resources are either project-scoped or user-scoped - project_scoped = (user_id is None and - key in per_project_resources) - user_scoped = (user_id is not None and - key not in per_project_resources) - if project_scoped or user_scoped: - syncable_resources.append(key) - return syncable_resources - def _get_quotas(self, context, resources, keys, project_id=None, user_id=None, project_quotas=None): """A helper method which retrieves the quotas for the specific @@ -662,227 +630,6 @@ class DbQuotaDriver(object): quotas=quotas_exceeded, usages={}, headroom=headroom) - def reserve(self, context, resources, deltas, expire=None, - project_id=None, user_id=None): - """Check quotas and reserve resources. - - For counting quotas--those quotas for which there is a usage - synchronization function--this method checks quotas against - current usage and the desired deltas. - - This method will raise a QuotaResourceUnknown exception if a - given resource is unknown or if it does not have a usage - synchronization function. - - If any of the proposed values is over the defined quota, an - OverQuota exception will be raised with the sorted list of the - resources which are too high. Otherwise, the method returns a - list of reservation UUIDs which were created. - - :param context: The request context, for access checks. - :param resources: A dictionary of the registered resources. - :param deltas: A dictionary of the proposed delta changes. - :param expire: An optional parameter specifying an expiration - time for the reservations. If it is a simple - number, it is interpreted as a number of - seconds and added to the current time; if it is - a datetime.timedelta object, it will also be - added to the current time. A datetime.datetime - object will be interpreted as the absolute - expiration time. If None is specified, the - default expiration time set by - --default-reservation-expire will be used (this - value will be treated as a number of seconds). - :param project_id: Specify the project_id if current context - is admin and admin wants to impact on - common user's tenant. - :param user_id: Specify the user_id if current context - is admin and admin wants to impact on - common user. - """ - _valid_method_call_check_resources(deltas, 'reserve', resources) - - # Set up the reservation expiration - if expire is None: - expire = CONF.quota.reservation_expire - if isinstance(expire, six.integer_types): - expire = datetime.timedelta(seconds=expire) - if isinstance(expire, datetime.timedelta): - expire = timeutils.utcnow() + expire - if not isinstance(expire, datetime.datetime): - raise exception.InvalidReservationExpiration(expire=expire) - - # If project_id is None, then we use the project_id in context - if project_id is None: - project_id = context.project_id - LOG.debug('Reserving resources using context.project_id: %s', - project_id) - # If user_id is None, then we use the project_id in context - if user_id is None: - user_id = context.user_id - LOG.debug('Reserving resources using context.user_id: %s', - user_id) - - LOG.debug('Attempting to reserve resources for project %(project_id)s ' - 'and user %(user_id)s. Deltas: %(deltas)s', - {'project_id': project_id, 'user_id': user_id, - 'deltas': deltas}) - - # Get the applicable quotas. - # NOTE(Vek): We're not worried about races at this point. - # Yes, the admin may be in the process of reducing - # quotas, but that's a pretty rare thing. - project_quotas = objects.Quotas.get_all_by_project(context, project_id) - LOG.debug('Quota limits for project %(project_id)s: ' - '%(project_quotas)s', {'project_id': project_id, - 'project_quotas': project_quotas}) - - quotas = self._get_quotas(context, resources, deltas.keys(), - project_id=project_id, - project_quotas=project_quotas) - LOG.debug('Quotas for project %(project_id)s after resource sync: ' - '%(quotas)s', {'project_id': project_id, 'quotas': quotas}) - user_quotas = self._get_quotas(context, resources, deltas.keys(), - project_id=project_id, - user_id=user_id, - project_quotas=project_quotas) - LOG.debug('Quotas for project %(project_id)s and user %(user_id)s ' - 'after resource sync: %(quotas)s', - {'project_id': project_id, 'user_id': user_id, - 'quotas': user_quotas}) - - # NOTE(Vek): Most of the work here has to be done in the DB - # API, because we have to do it in a transaction, - # which means access to the session. Since the - # session isn't available outside the DBAPI, we - # have to do the work there. - return db.quota_reserve(context, resources, quotas, user_quotas, - deltas, expire, - CONF.quota.until_refresh, CONF.quota.max_age, - project_id=project_id, user_id=user_id) - - def commit(self, context, reservations, project_id=None, user_id=None): - """Commit reservations. - - :param context: The request context, for access checks. - :param reservations: A list of the reservation UUIDs, as - returned by the reserve() method. - :param project_id: Specify the project_id if current context - is admin and admin wants to impact on - common user's tenant. - :param user_id: Specify the user_id if current context - is admin and admin wants to impact on - common user. - """ - # If project_id is None, then we use the project_id in context - if project_id is None: - project_id = context.project_id - # If user_id is None, then we use the user_id in context - if user_id is None: - user_id = context.user_id - - db.reservation_commit(context, reservations, project_id=project_id, - user_id=user_id) - - def rollback(self, context, reservations, project_id=None, user_id=None): - """Roll back reservations. - - :param context: The request context, for access checks. - :param reservations: A list of the reservation UUIDs, as - returned by the reserve() method. - :param project_id: Specify the project_id if current context - is admin and admin wants to impact on - common user's tenant. - :param user_id: Specify the user_id if current context - is admin and admin wants to impact on - common user. - """ - # If project_id is None, then we use the project_id in context - if project_id is None: - project_id = context.project_id - # If user_id is None, then we use the user_id in context - if user_id is None: - user_id = context.user_id - - db.reservation_rollback(context, reservations, project_id=project_id, - user_id=user_id) - - def usage_reset(self, context, resources): - """Reset the usage records for a particular user on a list of - resources. This will force that user's usage records to be - refreshed the next time a reservation is made. - - Note: this does not affect the currently outstanding - reservations the user has; those reservations must be - committed or rolled back (or expired). - - :param context: The request context, for access checks. - :param resources: A list of the resource names for which the - usage must be reset. - """ - - # We need an elevated context for the calls to - # quota_usage_update() - elevated = context.elevated() - - for resource in resources: - try: - # Reset the usage to -1, which will force it to be - # refreshed - db.quota_usage_update(elevated, context.project_id, - context.user_id, - resource, in_use=-1) - except exception.QuotaUsageNotFound: - # That means it'll be refreshed anyway - pass - - def usage_refresh(self, context, resources, project_id=None, - user_id=None, resource_names=None): - """Refresh the usage records for a particular project and user - on a list of resources. This will force usage records to be - sync'd immediately to the actual usage. - - This method will raise a QuotaUsageRefreshNotAllowed exception if a - usage refresh is not allowed on a resource for the given project - or user. - - :param context: The request context, for access checks. - :param resources: A dictionary of the registered resources. - :param project_id: Optional: Project whose resources to - refresh. If not set, then the project_id - is taken from the context. - :param user_id: Optional: User whose resources to refresh. - If not set, then the user_id is taken from the - context. - :param resources_names: Optional: A list of the resource names - for which the usage must be refreshed. - If not specified, then all the usages - for the project and user will be refreshed. - """ - - if project_id is None: - project_id = context.project_id - if user_id is None: - user_id = context.user_id - - syncable_resources = self._get_syncable_resources(resources, user_id) - - if resource_names: - for res_name in resource_names: - if res_name not in syncable_resources: - raise exception.QuotaUsageRefreshNotAllowed( - resource=res_name, - project_id=project_id, - user_id=user_id, - syncable=syncable_resources) - else: - resource_names = syncable_resources - - return db.quota_usage_refresh(context, resources, resource_names, - CONF.quota.until_refresh, - CONF.quota.max_age, - project_id=project_id, user_id=user_id) - def destroy_all_by_project_and_user(self, context, project_id, user_id): """Destroy all quotas associated with a project and user. @@ -1092,117 +839,6 @@ class NoopQuotaDriver(object): """ pass - def reserve(self, context, resources, deltas, expire=None, - project_id=None, user_id=None): - """Check quotas and reserve resources. - - For counting quotas--those quotas for which there is a usage - synchronization function--this method checks quotas against - current usage and the desired deltas. - - This method will raise a QuotaResourceUnknown exception if a - given resource is unknown or if it does not have a usage - synchronization function. - - If any of the proposed values is over the defined quota, an - OverQuota exception will be raised with the sorted list of the - resources which are too high. Otherwise, the method returns a - list of reservation UUIDs which were created. - - :param context: The request context, for access checks. - :param resources: A dictionary of the registered resources. - :param deltas: A dictionary of the proposed delta changes. - :param expire: An optional parameter specifying an expiration - time for the reservations. If it is a simple - number, it is interpreted as a number of - seconds and added to the current time; if it is - a datetime.timedelta object, it will also be - added to the current time. A datetime.datetime - object will be interpreted as the absolute - expiration time. If None is specified, the - default expiration time set by - --default-reservation-expire will be used (this - value will be treated as a number of seconds). - :param project_id: Specify the project_id if current context - is admin and admin wants to impact on - common user's tenant. - :param user_id: Specify the user_id if current context - is admin and admin wants to impact on - common user. - """ - return [] - - def commit(self, context, reservations, project_id=None, user_id=None): - """Commit reservations. - - :param context: The request context, for access checks. - :param reservations: A list of the reservation UUIDs, as - returned by the reserve() method. - :param project_id: Specify the project_id if current context - is admin and admin wants to impact on - common user's tenant. - :param user_id: Specify the user_id if current context - is admin and admin wants to impact on - common user. - """ - pass - - def rollback(self, context, reservations, project_id=None, user_id=None): - """Roll back reservations. - - :param context: The request context, for access checks. - :param reservations: A list of the reservation UUIDs, as - returned by the reserve() method. - :param project_id: Specify the project_id if current context - is admin and admin wants to impact on - common user's tenant. - :param user_id: Specify the user_id if current context - is admin and admin wants to impact on - common user. - """ - pass - - def usage_reset(self, context, resources): - """Reset the usage records for a particular user on a list of - resources. This will force that user's usage records to be - refreshed the next time a reservation is made. - - Note: this does not affect the currently outstanding - reservations the user has; those reservations must be - committed or rolled back (or expired). - - :param context: The request context, for access checks. - :param resources: A list of the resource names for which the - usage must be reset. - """ - pass - - def usage_refresh(self, context, resources, project_id=None, user_id=None, - resource_names=None): - """Refresh the usage records for a particular project and user - on a list of resources. This will force usage records to be - sync'd immediately to the actual usage. - - This method will raise a QuotaUsageRefreshNotAllowed exception if a - usage refresh is not allowed on a resource for the given project - or user. - - :param context: The request context, for access checks. - :param resources: A dictionary of the registered resources. - :param project_id: Optional: Project whose resources to - refresh. If not set, then the project_id - is taken from the context. - :param user_id: Optional: User whose resources to refresh. - If not set, then the user_id is taken from the - context. - :param resources_names: Optional: A list of the resource names - for which the usage must be refreshed. - If not specified, then all the usages - for the project and user will be refreshed. - """ - - pass - def destroy_all_by_project_and_user(self, context, project_id, user_id): """Destroy all quotas associated with a project and user. @@ -1294,44 +930,6 @@ class BaseResource(object): return CONF.quota[self.flag] if self.flag else -1 -class ReservableResource(BaseResource): - """Describe a reservable resource.""" - valid_method = 'reserve' - - def __init__(self, name, sync, flag=None): - """Initializes a ReservableResource. - - Reservable resources are those resources which directly - correspond to objects in the database, i.e., instances, - cores, etc. - - Usage synchronization function must be associated with each - object. This function will be called to determine the current - counts of one or more resources. This association is done in - database backend. - - The usage synchronization function will be passed three - arguments: an admin context, the project ID, and an opaque - session object, which should in turn be passed to the - underlying database function. Synchronization functions - should return a dictionary mapping resource names to the - current in_use count for those resources; more than one - resource and resource count may be returned. Note that - synchronization functions may be associated with more than one - ReservableResource. - - :param name: The name of the resource, i.e., "volumes". - :param sync: A dbapi methods name which returns a dictionary - to resynchronize the in_use count for one or more - resources, as described above. - :param flag: The name of the flag or configuration option - which specifies the default value of the quota - for this resource. - """ - super(ReservableResource, self).__init__(name, flag=flag) - self.sync = sync - - class AbsoluteResource(BaseResource): """Describe a resource that does not correspond to database objects.""" valid_method = 'check' @@ -1617,141 +1215,6 @@ class QuotaEngine(object): context, self._resources, project_values=project_values, user_values=user_values, project_id=project_id, user_id=user_id) - def reserve(self, context, expire=None, project_id=None, user_id=None, - **deltas): - """Check quotas and reserve resources. - - For counting quotas--those quotas for which there is a usage - synchronization function--this method checks quotas against - current usage and the desired deltas. The deltas are given as - keyword arguments, and current usage and other reservations - are factored into the quota check. - - This method will raise a QuotaResourceUnknown exception if a - given resource is unknown or if it does not have a usage - synchronization function. - - If any of the proposed values is over the defined quota, an - OverQuota exception will be raised with the sorted list of the - resources which are too high. Otherwise, the method returns a - list of reservation UUIDs which were created. - - :param context: The request context, for access checks. - :param expire: An optional parameter specifying an expiration - time for the reservations. If it is a simple - number, it is interpreted as a number of - seconds and added to the current time; if it is - a datetime.timedelta object, it will also be - added to the current time. A datetime.datetime - object will be interpreted as the absolute - expiration time. If None is specified, the - default expiration time set by - --default-reservation-expire will be used (this - value will be treated as a number of seconds). - :param project_id: Specify the project_id if current context - is admin and admin wants to impact on - common user's tenant. - """ - - reservations = self._driver.reserve(context, self._resources, deltas, - expire=expire, - project_id=project_id, - user_id=user_id) - - LOG.debug("Created reservations %s", reservations) - - return reservations - - def commit(self, context, reservations, project_id=None, user_id=None): - """Commit reservations. - - :param context: The request context, for access checks. - :param reservations: A list of the reservation UUIDs, as - returned by the reserve() method. - :param project_id: Specify the project_id if current context - is admin and admin wants to impact on - common user's tenant. - """ - - try: - self._driver.commit(context, reservations, project_id=project_id, - user_id=user_id) - except Exception: - # NOTE(Vek): Ignoring exceptions here is safe, because the - # usage resynchronization and the reservation expiration - # mechanisms will resolve the issue. The exception is - # logged, however, because this is less than optimal. - LOG.exception(_LE("Failed to commit reservations %s"), - reservations) - return - LOG.debug("Committed reservations %s", reservations) - - def rollback(self, context, reservations, project_id=None, user_id=None): - """Roll back reservations. - - :param context: The request context, for access checks. - :param reservations: A list of the reservation UUIDs, as - returned by the reserve() method. - :param project_id: Specify the project_id if current context - is admin and admin wants to impact on - common user's tenant. - """ - - try: - self._driver.rollback(context, reservations, project_id=project_id, - user_id=user_id) - except Exception: - # NOTE(Vek): Ignoring exceptions here is safe, because the - # usage resynchronization and the reservation expiration - # mechanisms will resolve the issue. The exception is - # logged, however, because this is less than optimal. - LOG.exception(_LE("Failed to roll back reservations %s"), - reservations) - return - LOG.debug("Rolled back reservations %s", reservations) - - def usage_reset(self, context, resources): - """Reset the usage records for a particular user on a list of - resources. This will force that user's usage records to be - refreshed the next time a reservation is made. - - Note: this does not affect the currently outstanding - reservations the user has; those reservations must be - committed or rolled back (or expired). - - :param context: The request context, for access checks. - :param resources: A list of the resource names for which the - usage must be reset. - """ - - self._driver.usage_reset(context, resources) - - def usage_refresh(self, context, project_id=None, user_id=None, - resource_names=None): - """Refresh the usage records for a particular project and user - on a list of resources. This will force usage records to be - sync'd immediately to the actual usage. - - This method will raise a QuotaUsageRefreshNotAllowed exception if a - usage refresh is not allowed on a resource for the given project - or user. - - :param context: The request context, for access checks. - :param project_id: Optional: Project whose resources to - refresh. If not set, then the project_id - is taken from the context. - :param user_id: Optional: User whose resources to refresh. - If not set, then the user_id is taken from the - context. - :param resources_names: Optional: A list of the resource names - for which the usage must be refreshed. - If not specified, then all the usages - for the project and user will be refreshed. - """ - - self._driver.usage_refresh(context, self._resources, project_id, - user_id, resource_names) - def destroy_all_by_project_and_user(self, context, project_id, user_id): """Destroy all quotas, usages, and reservations associated with a project and user. diff --git a/nova/tests/unit/db/test_db_api.py b/nova/tests/unit/db/test_db_api.py index 1cd9edb281c0..dec4a2343ba6 100644 --- a/nova/tests/unit/db/test_db_api.py +++ b/nova/tests/unit/db/test_db_api.py @@ -61,7 +61,6 @@ from nova.db.sqlalchemy import utils as db_utils from nova import exception from nova import objects from nova.objects import fields -from nova import quota from nova import test from nova.tests import fixtures as nova_fixtures from nova.tests.unit import fake_console_auth_token @@ -116,23 +115,10 @@ def _make_compute_node(host, node, hv_type, service_id): return compute_node_dict -def _quota_reserve(context, project_id, user_id): - """Create sample Quota, QuotaUsage and Reservation objects. - - There is no method db.quota_usage_create(), so we have to use - db.quota_reserve() for creating QuotaUsage objects. - - Returns reservations uuids. - - """ - def get_sync(resource, usage): - def sync(elevated, project_id, user_id): - return {resource: usage} - return sync +def _quota_create(context, project_id, user_id): + """Create sample Quota objects.""" quotas = {} user_quotas = {} - resources = {} - deltas = {} for i in range(3): resource = 'resource%d' % i if i == 2: @@ -149,16 +135,6 @@ def _quota_reserve(context, project_id, user_id): user_quotas[resource] = db.quota_create(context, project_id, resource, i + 1, user_id=user_id).hard_limit - sync_name = '_sync_%s' % resource - resources[resource] = quota.ReservableResource( - resource, sync_name, 'quota_res_%d' % i) - deltas[resource] = i - setattr(sqlalchemy_api, sync_name, get_sync(resource, i)) - sqlalchemy_api.QUOTA_SYNC_FUNCTIONS[sync_name] = getattr( - sqlalchemy_api, sync_name) - return db.quota_reserve(context, resources, quotas, user_quotas, deltas, - timeutils.utcnow(), CONF.quota.until_refresh, - datetime.timedelta(days=1), project_id, user_id) class DbTestCase(test.TestCase): @@ -1814,147 +1790,6 @@ class InstanceSystemMetadataTestCase(test.TestCase): {'key': 'value'}, True) -class RefreshUsageTestCase(test.TestCase): - """Tests for the db.api.quota_usage_refresh method. """ - - def setUp(self): - super(RefreshUsageTestCase, self).setUp() - self.ctxt = context.get_admin_context() - self.project_id = 'project1' - self.user_id = 'user1' - - def _quota_refresh(self, keys): - """Refresh the in_use count on the QuotaUsage objects. - The QuotaUsage objects are created if they don't exist. - """ - def get_sync(resource, usage): - def sync(elevated, project_id, user_id): - return {resource: usage} - return sync - - resources = {} - for i in range(4): - resource = 'resource%d' % i - if i == 2: - # test for project level resources - resource = 'fixed_ips' - if i == 3: - # test for project level resources - resource = 'floating_ips' - - sync_name = '_sync_%s' % resource - resources[resource] = quota.ReservableResource( - resource, sync_name, 'quota_res_%d' % i) - setattr(sqlalchemy_api, sync_name, get_sync(resource, i + 1)) - sqlalchemy_api.QUOTA_SYNC_FUNCTIONS[sync_name] = getattr( - sqlalchemy_api, sync_name) - - db.quota_usage_refresh(self.ctxt, resources, keys, - until_refresh=3, - max_age=0, - project_id=self.project_id, - user_id=self.user_id) - - def _compare_resource_usages(self, keys, expected, project_id, - user_id = None): - for key in keys: - actual = db.quota_usage_get(self.ctxt, project_id, key, user_id) - self.assertEqual(expected['project_id'], actual.project_id) - self.assertEqual(expected['user_id'], actual.user_id) - self.assertEqual(key, actual.resource) - self.assertEqual(expected[key]['in_use'], actual.in_use) - self.assertEqual(expected[key]['reserved'], actual.reserved) - self.assertEqual(expected[key]['until_refresh'], - actual.until_refresh) - - def test_refresh_created_project_usages(self): - # The refresh will create the usages and then sync - # in_use from 0 to 3 for fixed_ips and 0 to 4 for floating_ips. - keys = ['fixed_ips', 'floating_ips'] - self._quota_refresh(keys) - expected = {'project_id': self.project_id, - # User ID will be none for per-project resources - 'user_id': None, - 'fixed_ips': {'in_use': 3, 'reserved': 0, - 'until_refresh': 3}, - 'floating_ips': {'in_use': 4, 'reserved': 0, - 'until_refresh': 3}} - self._compare_resource_usages(keys, expected, self.project_id, - self.user_id) - - def test_refresh_created_user_usages(self): - # The refresh will create the usages and then sync - # in_use from 0 to 1 for resource0 and 0 to 2 for resource1. - keys = ['resource0', 'resource1'] - self._quota_refresh(keys) - expected = {'project_id': self.project_id, - 'user_id': self.user_id, - 'resource0': {'in_use': 1, 'reserved': 0, - 'until_refresh': 3}, - 'resource1': {'in_use': 2, 'reserved': 0, - 'until_refresh': 3}} - self._compare_resource_usages(keys, expected, self.project_id, - self.user_id) - - -class ReservationTestCase(test.TestCase, ModelsObjectComparatorMixin): - - """Tests for db.api.reservation_* methods.""" - - def setUp(self): - super(ReservationTestCase, self).setUp() - self.ctxt = context.get_admin_context() - - self.reservations = _quota_reserve(self.ctxt, 'project1', 'user1') - usage = db.quota_usage_get(self.ctxt, 'project1', 'resource1', 'user1') - - self.values = {'uuid': 'sample-uuid', - 'project_id': 'project1', - 'user_id': 'user1', - 'resource': 'resource1', - 'delta': 42, - 'expire': timeutils.utcnow() + datetime.timedelta(days=1), - 'usage': {'id': usage.id}} - - def test_reservation_commit(self): - expected = {'project_id': 'project1', 'user_id': 'user1', - 'resource0': {'reserved': 0, 'in_use': 0}, - 'resource1': {'reserved': 1, 'in_use': 1}, - 'fixed_ips': {'reserved': 2, 'in_use': 2}} - self.assertEqual(expected, db.quota_usage_get_all_by_project_and_user( - self.ctxt, 'project1', 'user1')) - _reservation_get(self.ctxt, self.reservations[0]) - db.reservation_commit(self.ctxt, self.reservations, 'project1', - 'user1') - self.assertRaises(exception.ReservationNotFound, - _reservation_get, self.ctxt, self.reservations[0]) - expected = {'project_id': 'project1', 'user_id': 'user1', - 'resource0': {'reserved': 0, 'in_use': 0}, - 'resource1': {'reserved': 0, 'in_use': 2}, - 'fixed_ips': {'reserved': 0, 'in_use': 4}} - self.assertEqual(expected, db.quota_usage_get_all_by_project_and_user( - self.ctxt, 'project1', 'user1')) - - def test_reservation_rollback(self): - expected = {'project_id': 'project1', 'user_id': 'user1', - 'resource0': {'reserved': 0, 'in_use': 0}, - 'resource1': {'reserved': 1, 'in_use': 1}, - 'fixed_ips': {'reserved': 2, 'in_use': 2}} - self.assertEqual(expected, db.quota_usage_get_all_by_project_and_user( - self.ctxt, 'project1', 'user1')) - _reservation_get(self.ctxt, self.reservations[0]) - db.reservation_rollback(self.ctxt, self.reservations, 'project1', - 'user1') - self.assertRaises(exception.ReservationNotFound, - _reservation_get, self.ctxt, self.reservations[0]) - expected = {'project_id': 'project1', 'user_id': 'user1', - 'resource0': {'reserved': 0, 'in_use': 0}, - 'resource1': {'reserved': 0, 'in_use': 1}, - 'fixed_ips': {'reserved': 0, 'in_use': 2}} - self.assertEqual(expected, db.quota_usage_get_all_by_project_and_user( - self.ctxt, 'project1', 'user1')) - - class SecurityGroupRuleTestCase(test.TestCase, ModelsObjectComparatorMixin): def setUp(self): super(SecurityGroupRuleTestCase, self).setUp() @@ -2286,23 +2121,6 @@ class SecurityGroupTestCase(test.TestCase, ModelsObjectComparatorMixin): self.assertEqual(1, len(security_groups)) self.assertEqual("default", security_groups[0]["name"]) - usage = db.quota_usage_get(self.ctxt, - self.ctxt.project_id, - 'security_groups', - self.ctxt.user_id) - self.assertEqual(1, usage.in_use) - - def test_security_group_ensure_default_until_refresh(self): - self.flags(until_refresh=2, group='quota') - self.ctxt.project_id = 'fake' - self.ctxt.user_id = 'fake' - db.security_group_ensure_default(self.ctxt) - usage = db.quota_usage_get(self.ctxt, - self.ctxt.project_id, - 'security_groups', - self.ctxt.user_id) - self.assertEqual(2, usage.until_refresh) - @mock.patch.object(db.sqlalchemy.api, '_security_group_get_by_names') def test_security_group_ensure_default_called_concurrently(self, sg_mock): # make sure NotFound is always raised here to trick Nova to insert the @@ -7521,151 +7339,23 @@ class QuotaTestCase(test.TestCase, ModelsObjectComparatorMixin): self.assertRaises(exception.ProjectQuotaNotFound, db.quota_get, self.ctxt, 'project1', 'resource1') - def test_quota_reserve_all_resources(self): - quotas = {} - deltas = {} - reservable_resources = {} - for i, resource in enumerate(quota.resources): - if isinstance(resource, quota.ReservableResource): - quotas[resource.name] = db.quota_create(self.ctxt, 'project1', - resource.name, - 100).hard_limit - deltas[resource.name] = i - reservable_resources[resource.name] = resource - - usages = {'instances': 3, 'cores': 6, 'ram': 9} - instances = [] - for i in range(3): - instances.append(db.instance_create(self.ctxt, - {'vcpus': 2, 'memory_mb': 3, - 'project_id': 'project1'})) - - usages['fixed_ips'] = 2 - network = db.network_create_safe(self.ctxt, {}) - for i in range(2): - address = '192.168.0.%d' % i - db.fixed_ip_create(self.ctxt, {'project_id': 'project1', - 'address': address, - 'network_id': network['id']}) - db.fixed_ip_associate(self.ctxt, address, - instances[0].uuid, network['id']) - - usages['floating_ips'] = 5 - for i in range(5): - db.floating_ip_create(self.ctxt, {'project_id': 'project1'}) - - usages['security_groups'] = 3 - for i in range(3): - db.security_group_create(self.ctxt, {'project_id': 'project1'}) - - usages['server_groups'] = 4 - for i in range(4): - db.instance_group_create(self.ctxt, {'uuid': str(i), - 'project_id': 'project1'}) - - reservations_uuids = db.quota_reserve(self.ctxt, reservable_resources, - quotas, quotas, deltas, None, - None, None, 'project1') - resources_names = list(reservable_resources.keys()) - for reservation_uuid in reservations_uuids: - reservation = _reservation_get(self.ctxt, reservation_uuid) - usage = db.quota_usage_get(self.ctxt, 'project1', - reservation.resource) - self.assertEqual(usage.in_use, usages[reservation.resource], - 'Resource: %s' % reservation.resource) - self.assertEqual(usage.reserved, deltas[reservation.resource]) - self.assertIn(reservation.resource, resources_names) - resources_names.remove(reservation.resource) - self.assertEqual(len(resources_names), 0) - def test_quota_destroy_all_by_project(self): - reservations = _quota_reserve(self.ctxt, 'project1', 'user1') + _quota_create(self.ctxt, 'project1', 'user1') db.quota_destroy_all_by_project(self.ctxt, 'project1') self.assertEqual(db.quota_get_all_by_project(self.ctxt, 'project1'), {'project_id': 'project1'}) self.assertEqual(db.quota_get_all_by_project_and_user(self.ctxt, 'project1', 'user1'), {'project_id': 'project1', 'user_id': 'user1'}) - self.assertEqual(db.quota_usage_get_all_by_project( - self.ctxt, 'project1'), - {'project_id': 'project1'}) - for r in reservations: - self.assertRaises(exception.ReservationNotFound, - _reservation_get, self.ctxt, r) def test_quota_destroy_all_by_project_and_user(self): - reservations = _quota_reserve(self.ctxt, 'project1', 'user1') + _quota_create(self.ctxt, 'project1', 'user1') db.quota_destroy_all_by_project_and_user(self.ctxt, 'project1', 'user1') self.assertEqual(db.quota_get_all_by_project_and_user(self.ctxt, 'project1', 'user1'), {'project_id': 'project1', 'user_id': 'user1'}) - self.assertEqual(db.quota_usage_get_all_by_project_and_user( - self.ctxt, 'project1', 'user1'), - {'project_id': 'project1', - 'user_id': 'user1', - 'fixed_ips': {'in_use': 2, 'reserved': 2}}) - for r in reservations: - self.assertRaises(exception.ReservationNotFound, - _reservation_get, self.ctxt, r) - - def test_quota_usage_get_nonexistent(self): - self.assertRaises(exception.QuotaUsageNotFound, db.quota_usage_get, - self.ctxt, 'p1', 'nonexitent_resource') - - def test_quota_usage_get(self): - _quota_reserve(self.ctxt, 'p1', 'u1') - quota_usage = db.quota_usage_get(self.ctxt, 'p1', 'resource0') - expected = {'resource': 'resource0', 'project_id': 'p1', - 'in_use': 0, 'reserved': 0, 'total': 0} - for key, value in expected.items(): - self.assertEqual(value, quota_usage[key]) - - def test_quota_usage_get_all_by_project(self): - _quota_reserve(self.ctxt, 'p1', 'u1') - expected = {'project_id': 'p1', - 'resource0': {'in_use': 0, 'reserved': 0}, - 'resource1': {'in_use': 1, 'reserved': 1}, - 'fixed_ips': {'in_use': 2, 'reserved': 2}} - self.assertEqual(expected, db.quota_usage_get_all_by_project( - self.ctxt, 'p1')) - - def test_quota_usage_get_all_by_project_and_user(self): - _quota_reserve(self.ctxt, 'p1', 'u1') - expected = {'project_id': 'p1', - 'user_id': 'u1', - 'resource0': {'in_use': 0, 'reserved': 0}, - 'resource1': {'in_use': 1, 'reserved': 1}, - 'fixed_ips': {'in_use': 2, 'reserved': 2}} - self.assertEqual(expected, db.quota_usage_get_all_by_project_and_user( - self.ctxt, 'p1', 'u1')) - - def test_get_project_user_quota_usages_in_order(self): - _quota_reserve(self.ctxt, 'p1', 'u1') - - @sqlalchemy_api.pick_context_manager_reader - def test(context): - with mock.patch.object(query.Query, 'order_by') as order_mock: - sqlalchemy_api._get_project_user_quota_usages( - context, 'p1', 'u1') - self.assertTrue(order_mock.called) - - test(self.ctxt) - - def test_quota_usage_update_nonexistent(self): - self.assertRaises(exception.QuotaUsageNotFound, db.quota_usage_update, - self.ctxt, 'p1', 'u1', 'resource', in_use=42) - - def test_quota_usage_update(self): - _quota_reserve(self.ctxt, 'p1', 'u1') - db.quota_usage_update(self.ctxt, 'p1', 'u1', 'resource0', in_use=42, - reserved=43) - quota_usage = db.quota_usage_get(self.ctxt, 'p1', 'resource0', 'u1') - expected = {'resource': 'resource0', 'project_id': 'p1', - 'user_id': 'u1', 'in_use': 42, 'reserved': 43, 'total': 85} - for key, value in expected.items(): - self.assertEqual(value, quota_usage[key]) def test_quota_create_exists(self): db.quota_create(self.ctxt, 'project1', 'resource1', 41) @@ -7673,144 +7363,6 @@ class QuotaTestCase(test.TestCase, ModelsObjectComparatorMixin): 'project1', 'resource1', 42) -class QuotaReserveNoDbTestCase(test.NoDBTestCase): - """Tests quota reserve/refresh operations using mock.""" - - def test_create_quota_usage_if_missing_not_created(self): - # Tests that QuotaUsage isn't created if it's already in user_usages. - resource = 'fake-resource' - project_id = 'fake-project' - user_id = 'fake_user' - session = mock.sentinel - quota_usage = mock.sentinel - user_usages = {resource: quota_usage} - with mock.patch.object(sqlalchemy_api, '_quota_usage_create') as quc: - self.assertFalse(sqlalchemy_api._create_quota_usage_if_missing( - user_usages, resource, None, - project_id, user_id, session)) - self.assertFalse(quc.called) - - def _test_create_quota_usage_if_missing_created(self, per_project_quotas): - # Tests that the QuotaUsage is created. - user_usages = {} - if per_project_quotas: - resource = sqlalchemy_api.PER_PROJECT_QUOTAS[0] - else: - resource = 'fake-resource' - project_id = 'fake-project' - user_id = 'fake_user' - session = mock.sentinel - quota_usage = mock.sentinel - with mock.patch.object(sqlalchemy_api, '_quota_usage_create', - return_value=quota_usage) as quc: - self.assertTrue(sqlalchemy_api._create_quota_usage_if_missing( - user_usages, resource, None, - project_id, user_id, session)) - self.assertEqual(quota_usage, user_usages[resource]) - # Now test if the QuotaUsage was created with a user_id or not. - if per_project_quotas: - quc.assert_called_once_with( - project_id, None, resource, 0, 0, None, session) - else: - quc.assert_called_once_with( - project_id, user_id, resource, 0, 0, None, session) - - def test_create_quota_usage_if_missing_created_per_project_quotas(self): - self._test_create_quota_usage_if_missing_created(True) - - def test_create_quota_usage_if_missing_created_user_quotas(self): - self._test_create_quota_usage_if_missing_created(False) - - def test_is_quota_refresh_needed_in_use(self): - # Tests when a quota refresh is needed based on the in_use value. - for in_use in range(-1, 1): - # We have to set until_refresh=None otherwise mock will give it - # a value which runs some code we don't want. - quota_usage = mock.MagicMock(in_use=in_use, until_refresh=None) - if in_use < 0: - self.assertTrue(sqlalchemy_api._is_quota_refresh_needed( - quota_usage, max_age=0)) - else: - self.assertFalse(sqlalchemy_api._is_quota_refresh_needed( - quota_usage, max_age=0)) - - def test_is_quota_refresh_needed_until_refresh_none(self): - quota_usage = mock.MagicMock(in_use=0, until_refresh=None) - self.assertFalse(sqlalchemy_api._is_quota_refresh_needed(quota_usage, - max_age=0)) - - def test_is_quota_refresh_needed_until_refresh_not_none(self): - # Tests different values for the until_refresh counter. - for until_refresh in range(3): - quota_usage = mock.MagicMock(in_use=0, until_refresh=until_refresh) - refresh = sqlalchemy_api._is_quota_refresh_needed(quota_usage, - max_age=0) - until_refresh -= 1 - if until_refresh <= 0: - self.assertTrue(refresh) - else: - self.assertFalse(refresh) - self.assertEqual(until_refresh, quota_usage.until_refresh) - - def test_refresh_quota_usages(self): - quota_usage = mock.Mock(spec=models.QuotaUsage) - quota_usage.in_use = 5 - quota_usage.until_refresh = None - sqlalchemy_api._refresh_quota_usages(quota_usage, until_refresh=5, - in_use=6) - self.assertEqual(6, quota_usage.in_use) - self.assertEqual(5, quota_usage.until_refresh) - - def test_calculate_overquota_no_delta(self): - deltas = {'foo': -1} - user_quotas = {'foo': 10} - overs = sqlalchemy_api._calculate_overquota({}, user_quotas, deltas, - {}, {}) - self.assertFalse(overs) - - def test_calculate_overquota_unlimited_user_quota(self): - deltas = {'foo': 1} - project_quotas = {'foo': -1} - user_quotas = {'foo': -1} - project_usages = {'foo': {'total': 10}} - user_usages = {'foo': {'total': 10}} - overs = sqlalchemy_api._calculate_overquota( - project_quotas, user_quotas, deltas, project_usages, user_usages) - self.assertFalse(overs) - - def test_calculate_overquota_unlimited_project_quota(self): - deltas = {'foo': 1} - project_quotas = {'foo': -1} - user_quotas = {'foo': 1} - project_usages = {'foo': {'total': 0}} - user_usages = {'foo': {'total': 0}} - overs = sqlalchemy_api._calculate_overquota( - project_quotas, user_quotas, deltas, project_usages, user_usages) - self.assertFalse(overs) - - def _test_calculate_overquota(self, resource, project_usages, user_usages): - deltas = {resource: 1} - project_quotas = {resource: 10} - user_quotas = {resource: 10} - overs = sqlalchemy_api._calculate_overquota( - project_quotas, user_quotas, deltas, project_usages, user_usages) - self.assertEqual(resource, overs[0]) - - def test_calculate_overquota_per_project_quota_overquota(self): - # In this test, user quotas are fine but project quotas are over. - resource = 'foo' - project_usages = {resource: {'total': 10}} - user_usages = {resource: {'total': 5}} - self._test_calculate_overquota(resource, project_usages, user_usages) - - def test_calculate_overquota_per_user_quota_overquota(self): - # In this test, project quotas are fine but user quotas are over. - resource = 'foo' - project_usages = {resource: {'total': 5}} - user_usages = {resource: {'total': 10}} - self._test_calculate_overquota(resource, project_usages, user_usages) - - class QuotaClassTestCase(test.TestCase, ModelsObjectComparatorMixin): def setUp(self): @@ -7867,11 +7419,6 @@ class QuotaClassTestCase(test.TestCase, ModelsObjectComparatorMixin): self.assertRaises(exception.QuotaClassNotFound, db.quota_class_update, self.ctxt, 'class name', 'resource', 42) - def test_refresh_quota_usages(self): - quota_usages = mock.Mock() - sqlalchemy_api._refresh_quota_usages(quota_usages, until_refresh=5, - in_use=6) - class S3ImageTestCase(test.TestCase): diff --git a/nova/tests/unit/objects/test_quotas.py b/nova/tests/unit/objects/test_quotas.py index 0f1f470e9e8a..c736a3738e14 100644 --- a/nova/tests/unit/objects/test_quotas.py +++ b/nova/tests/unit/objects/test_quotas.py @@ -54,93 +54,6 @@ class _TestQuotasObject(object): self.instance = fake_instance.fake_db_instance( project_id='fake_proj2', user_id='fake_user2') - def test_from_reservations(self): - fake_reservations = ['1', '2'] - quotas = quotas_obj.Quotas.from_reservations( - self.context, fake_reservations) - self.assertEqual(self.context, quotas._context) - self.assertEqual(fake_reservations, quotas.reservations) - self.assertIsNone(quotas.project_id) - self.assertIsNone(quotas.user_id) - - def test_from_reservations_bogus(self): - fake_reservations = [_TestQuotasObject, _TestQuotasObject] - self.assertRaises(ValueError, - quotas_obj.Quotas.from_reservations, - self.context, fake_reservations) - - def test_from_reservations_instance(self): - fake_reservations = ['1', '2'] - quotas = quotas_obj.Quotas.from_reservations( - self.context, fake_reservations, - instance=self.instance) - self.assertEqual(self.context, quotas._context) - self.assertEqual(fake_reservations, quotas.reservations) - self.assertEqual('fake_proj1', quotas.project_id) - self.assertEqual('fake_user2', quotas.user_id) - - def test_from_reservations_instance_admin(self): - fake_reservations = ['1', '2'] - elevated = self.context.elevated() - quotas = quotas_obj.Quotas.from_reservations( - elevated, fake_reservations, - instance=self.instance) - self.assertEqual(elevated, quotas._context) - self.assertEqual(fake_reservations, quotas.reservations) - self.assertEqual('fake_proj2', quotas.project_id) - self.assertEqual('fake_user2', quotas.user_id) - - @mock.patch.object(QUOTAS, 'reserve') - def test_reserve(self, reserve_mock): - fake_reservations = ['1', '2'] - quotas = quotas_obj.Quotas(context=self.context) - - reserve_mock.return_value = fake_reservations - - quotas.reserve(expire='expire', - project_id='project_id', user_id='user_id', - moo='cow') - self.assertEqual(self.context, quotas._context) - self.assertEqual(fake_reservations, quotas.reservations) - self.assertEqual('project_id', quotas.project_id) - self.assertEqual('user_id', quotas.user_id) - reserve_mock.assert_called_once_with( - self.context, expire='expire', project_id='project_id', - user_id='user_id', moo='cow') - - @mock.patch.object(QUOTAS, 'commit') - def test_commit(self, commit_mock): - fake_reservations = ['1', '2'] - quotas = quotas_obj.Quotas.from_reservations( - self.context, fake_reservations) - quotas.commit() - self.assertIsNone(quotas.reservations) - commit_mock.assert_called_once_with( - self.context, fake_reservations, project_id=None, user_id=None) - - @mock.patch.object(QUOTAS, 'commit') - def test_commit_none_reservations(self, commit_mock): - quotas = quotas_obj.Quotas.from_reservations(self.context, None) - quotas.commit() - self.assertFalse(commit_mock.called) - - @mock.patch.object(QUOTAS, 'rollback') - def test_rollback(self, rollback_mock): - fake_reservations = ['1', '2'] - quotas = quotas_obj.Quotas.from_reservations( - self.context, fake_reservations) - - quotas.rollback() - self.assertIsNone(quotas.reservations) - rollback_mock.assert_called_once_with( - self.context, fake_reservations, project_id=None, user_id=None) - - @mock.patch.object(QUOTAS, 'rollback') - def test_rollback_none_reservations(self, rollback_mock): - quotas = quotas_obj.Quotas.from_reservations(self.context, None) - quotas.rollback() - self.assertFalse(rollback_mock.called) - @mock.patch('nova.db.quota_get', side_effect=exception.QuotaNotFound) @mock.patch('nova.objects.Quotas._create_limit_in_db') def test_create_limit(self, mock_create, mock_get): diff --git a/nova/tests/unit/test_quota.py b/nova/tests/unit/test_quota.py index d55e12ff4496..95c287972e83 100644 --- a/nova/tests/unit/test_quota.py +++ b/nova/tests/unit/test_quota.py @@ -340,23 +340,6 @@ class FakeDriver(object): self.called.append(('limit_check_project_and_user', context, resources, project_values, user_values, project_id, user_id)) - def reserve(self, context, resources, deltas, expire=None, - project_id=None, user_id=None): - self.called.append(('reserve', context, resources, deltas, - expire, project_id, user_id)) - return self.reservations - - def commit(self, context, reservations, project_id=None, user_id=None): - self.called.append(('commit', context, reservations, project_id, - user_id)) - - def rollback(self, context, reservations, project_id=None, user_id=None): - self.called.append(('rollback', context, reservations, project_id, - user_id)) - - def usage_reset(self, context, resources): - self.called.append(('usage_reset', context, resources)) - def destroy_all_by_project_and_user(self, context, project_id, user_id): self.called.append(('destroy_all_by_project_and_user', context, project_id, user_id)) @@ -488,7 +471,7 @@ class BaseResourceTestCase(test.TestCase): quota._valid_method_call_check_resources, resources, 'check', quota.QUOTAS._resources) - def test_valid_method_call_check_wrong_method_reserve(self): + def test_valid_method_call_check_wrong_method(self): resources = {'key_pairs': 1} engine_resources = {'key_pairs': quota.CountableResource('key_pairs', None, @@ -496,17 +479,7 @@ class BaseResourceTestCase(test.TestCase): self.assertRaises(exception.InvalidQuotaMethodUsage, quota._valid_method_call_check_resources, - resources, 'reserve', engine_resources) - - def test_valid_method_call_check_wrong_method_check(self): - resources = {'instances': 1} - engine_resources = {'instances': quota.ReservableResource('instances', - None, - 'instances')} - - self.assertRaises(exception.InvalidQuotaMethodUsage, - quota._valid_method_call_check_resources, - resources, 'check', engine_resources) + resources, 'bogus', engine_resources) class QuotaEngineTestCase(test.TestCase): @@ -734,84 +707,6 @@ class QuotaEngineTestCase(test.TestCase): None, None)], driver.called) - def test_reserve(self): - context = FakeContext(None, None) - driver = FakeDriver(reservations=[ - 'resv-01', 'resv-02', 'resv-03', 'resv-04', - ]) - quota_obj = self._make_quota_obj(driver) - result1 = quota_obj.reserve(context, test_resource1=4, - test_resource2=3, test_resource3=2, - test_resource4=1) - result2 = quota_obj.reserve(context, expire=3600, - test_resource1=1, test_resource2=2, - test_resource3=3, test_resource4=4) - result3 = quota_obj.reserve(context, project_id='fake_project', - test_resource1=1, test_resource2=2, - test_resource3=3, test_resource4=4) - - self.assertEqual(driver.called, [ - ('reserve', context, quota_obj._resources, dict( - test_resource1=4, - test_resource2=3, - test_resource3=2, - test_resource4=1, - ), None, None, None), - ('reserve', context, quota_obj._resources, dict( - test_resource1=1, - test_resource2=2, - test_resource3=3, - test_resource4=4, - ), 3600, None, None), - ('reserve', context, quota_obj._resources, dict( - test_resource1=1, - test_resource2=2, - test_resource3=3, - test_resource4=4, - ), None, 'fake_project', None), - ]) - self.assertEqual(result1, [ - 'resv-01', 'resv-02', 'resv-03', 'resv-04', - ]) - self.assertEqual(result2, [ - 'resv-01', 'resv-02', 'resv-03', 'resv-04', - ]) - self.assertEqual(result3, [ - 'resv-01', 'resv-02', 'resv-03', 'resv-04', - ]) - - def test_commit(self): - context = FakeContext(None, None) - driver = FakeDriver() - quota_obj = self._make_quota_obj(driver) - quota_obj.commit(context, ['resv-01', 'resv-02', 'resv-03']) - - self.assertEqual(driver.called, [ - ('commit', context, ['resv-01', 'resv-02', 'resv-03'], None, - None), - ]) - - def test_rollback(self): - context = FakeContext(None, None) - driver = FakeDriver() - quota_obj = self._make_quota_obj(driver) - quota_obj.rollback(context, ['resv-01', 'resv-02', 'resv-03']) - - self.assertEqual(driver.called, [ - ('rollback', context, ['resv-01', 'resv-02', 'resv-03'], None, - None), - ]) - - def test_usage_reset(self): - context = FakeContext(None, None) - driver = FakeDriver() - quota_obj = self._make_quota_obj(driver) - quota_obj.usage_reset(context, ['res1', 'res2', 'res3']) - - self.assertEqual(driver.called, [ - ('usage_reset', context, ['res1', 'res2', 'res3']), - ]) - def test_destroy_all_by_project_and_user(self): context = FakeContext(None, None) driver = FakeDriver()