Merge "Remove old-style quotas code"

This commit is contained in:
Zuul 2017-12-01 02:53:17 +00:00 committed by Gerrit Code Review
commit 863c13ef5c
2 changed files with 0 additions and 588 deletions
nova/db
api.py
sqlalchemy

@ -1198,74 +1198,6 @@ def quota_class_update(context, class_name, resource, limit):
###################
def quota_usage_get(context, project_id, resource, user_id=None):
"""Retrieve a quota usage or raise if it does not exist."""
return IMPL.quota_usage_get(context, project_id, resource, user_id=user_id)
def quota_usage_get_all_by_project_and_user(context, project_id, user_id):
"""Retrieve all usage associated with a given resource."""
return IMPL.quota_usage_get_all_by_project_and_user(context,
project_id, user_id)
def quota_usage_get_all_by_project(context, project_id):
"""Retrieve all usage associated with a given resource."""
return IMPL.quota_usage_get_all_by_project(context, project_id)
def quota_usage_update(context, project_id, user_id, resource, **kwargs):
"""Update a quota usage or raise if it does not exist."""
return IMPL.quota_usage_update(context, project_id, user_id, resource,
**kwargs)
def quota_usage_refresh(context, resources, keys, until_refresh, max_age,
project_id=None, user_id=None):
"""Refresh the quota usages.
:param context: The request context, for access checks.
:param resources: A dictionary of the registered resources.
:param keys: Names of the resources whose usage is to be refreshed.
:param until_refresh: The until_refresh configuration value.
:param max_age: The max_age configuration value.
:param project_id: (Optional) The project_id containing the usages
to be refreshed. Defaults to the project_id
in the context.
:param user_id: (Optional) The user_id containing the usages
to be refreshed. Defaults to the user_id
in the context.
"""
return IMPL.quota_usage_refresh(context, resources, keys, until_refresh,
max_age, project_id=project_id, user_id=user_id)
###################
def quota_reserve(context, resources, quotas, user_quotas, deltas, expire,
until_refresh, max_age, project_id=None, user_id=None):
"""Check quotas and create appropriate reservations."""
return IMPL.quota_reserve(context, resources, quotas, user_quotas, deltas,
expire, until_refresh, max_age,
project_id=project_id, user_id=user_id)
def reservation_commit(context, reservations, project_id=None, user_id=None):
"""Commit quota reservations."""
return IMPL.reservation_commit(context, reservations,
project_id=project_id,
user_id=user_id)
def reservation_rollback(context, reservations, project_id=None, user_id=None):
"""Roll back quota reservations."""
return IMPL.reservation_rollback(context, reservations,
project_id=project_id,
user_id=user_id)
def quota_destroy_all_by_project_and_user(context, project_id, user_id):
"""Destroy all quotas associated with a given project and user."""
return IMPL.quota_destroy_all_by_project_and_user(context,

@ -352,37 +352,6 @@ def convert_objects_related_datetimes(values, *datetime_keys):
return values
def _sync_instances(context, project_id, user_id):
return dict(zip(('instances', 'cores', 'ram'),
_instance_data_get_for_user(context, project_id, user_id)))
def _sync_floating_ips(context, project_id, user_id):
return dict(floating_ips=_floating_ip_count_by_project(
context, project_id))
def _sync_fixed_ips(context, project_id, user_id):
return dict(fixed_ips=_fixed_ip_count_by_project(context, project_id))
def _sync_security_groups(context, project_id, user_id):
return dict(security_groups=_security_group_count_by_project_and_user(
context, project_id, user_id))
def _sync_server_groups(context, project_id, user_id):
return dict(server_groups=_instance_group_count_by_project_and_user(
context, project_id, user_id))
QUOTA_SYNC_FUNCTIONS = {
'_sync_instances': _sync_instances,
'_sync_floating_ips': _sync_floating_ips,
'_sync_fixed_ips': _sync_fixed_ips,
'_sync_security_groups': _sync_security_groups,
'_sync_server_groups': _sync_server_groups,
}
###################
@ -3595,481 +3564,6 @@ def quota_class_update(context, class_name, resource, limit):
###################
@require_context
@pick_context_manager_reader
def quota_usage_get(context, project_id, resource, user_id=None):
query = model_query(context, models.QuotaUsage, read_deleted="no").\
filter_by(project_id=project_id).\
filter_by(resource=resource)
if user_id:
if resource not in PER_PROJECT_QUOTAS:
result = query.filter_by(user_id=user_id).first()
else:
result = query.filter_by(user_id=None).first()
else:
result = query.first()
if not result:
raise exception.QuotaUsageNotFound(project_id=project_id)
return result
def _quota_usage_get_all(context, project_id, user_id=None):
query = model_query(context, models.QuotaUsage, read_deleted="no").\
filter_by(project_id=project_id)
result = {'project_id': project_id}
if user_id:
query = query.filter(or_(models.QuotaUsage.user_id == user_id,
models.QuotaUsage.user_id == null()))
result['user_id'] = user_id
rows = query.all()
for row in rows:
if row.resource in result:
result[row.resource]['in_use'] += row.in_use
result[row.resource]['reserved'] += row.reserved
else:
result[row.resource] = dict(in_use=row.in_use,
reserved=row.reserved)
return result
@require_context
@pick_context_manager_reader
def quota_usage_get_all_by_project_and_user(context, project_id, user_id):
return _quota_usage_get_all(context, project_id, user_id=user_id)
@require_context
@pick_context_manager_reader
def quota_usage_get_all_by_project(context, project_id):
return _quota_usage_get_all(context, project_id)
def _quota_usage_create(project_id, user_id, resource, in_use,
reserved, until_refresh, session):
quota_usage_ref = models.QuotaUsage()
quota_usage_ref.project_id = project_id
quota_usage_ref.user_id = user_id
quota_usage_ref.resource = resource
quota_usage_ref.in_use = in_use
quota_usage_ref.reserved = reserved
quota_usage_ref.until_refresh = until_refresh
# updated_at is needed for judgement of max_age
quota_usage_ref.updated_at = timeutils.utcnow()
quota_usage_ref.save(session)
return quota_usage_ref
@pick_context_manager_writer
def quota_usage_update(context, project_id, user_id, resource, **kwargs):
updates = {}
for key in ['in_use', 'reserved', 'until_refresh']:
if key in kwargs:
updates[key] = kwargs[key]
result = model_query(context, models.QuotaUsage, read_deleted="no").\
filter_by(project_id=project_id).\
filter_by(resource=resource).\
filter(or_(models.QuotaUsage.user_id == user_id,
models.QuotaUsage.user_id == null())).\
update(updates)
if not result:
raise exception.QuotaUsageNotFound(project_id=project_id)
###################
def _reservation_create(uuid, usage, project_id, user_id, resource,
delta, expire, session):
reservation_ref = models.Reservation()
reservation_ref.uuid = uuid
reservation_ref.usage_id = usage['id']
reservation_ref.project_id = project_id
reservation_ref.user_id = user_id
reservation_ref.resource = resource
reservation_ref.delta = delta
reservation_ref.expire = expire
reservation_ref.save(session)
return reservation_ref
###################
# NOTE(johannes): The quota code uses SQL locking to ensure races don't
# cause under or over counting of resources. To avoid deadlocks, this
# code always acquires the lock on quota_usages before acquiring the lock
# on reservations.
def _get_project_user_quota_usages(context, project_id, user_id):
rows = model_query(context, models.QuotaUsage,
read_deleted="no").\
filter_by(project_id=project_id).\
order_by(models.QuotaUsage.id.asc()).\
with_lockmode('update').\
all()
proj_result = dict()
user_result = dict()
# Get the total count of in_use,reserved
for row in rows:
proj_result.setdefault(row.resource,
dict(in_use=0, reserved=0, total=0))
proj_result[row.resource]['in_use'] += row.in_use
proj_result[row.resource]['reserved'] += row.reserved
proj_result[row.resource]['total'] += (row.in_use + row.reserved)
if row.user_id is None or row.user_id == user_id:
user_result[row.resource] = row
return proj_result, user_result
def _create_quota_usage_if_missing(user_usages, resource, until_refresh,
project_id, user_id, session):
"""Creates a QuotaUsage record and adds to user_usages if not present.
:param user_usages: dict of resource keys to QuotaUsage records. This is
updated if resource is not in user_usages yet or
until_refresh is not None.
:param resource: The resource being checked for quota usage.
:param until_refresh: Count of reservations until usage is refreshed,
int or None
:param project_id: The project being checked for quota usage.
:param user_id: The user being checked for quota usage.
:param session: DB session holding a transaction lock.
:return: True if a new QuotaUsage record was created and added
to user_usages, False otherwise.
"""
new_usage = None
if resource not in user_usages:
user_id_to_use = user_id
if resource in PER_PROJECT_QUOTAS:
user_id_to_use = None
new_usage = _quota_usage_create(project_id, user_id_to_use, resource,
0, 0, until_refresh or None, session)
user_usages[resource] = new_usage
return new_usage is not None
def _is_quota_refresh_needed(quota_usage, max_age):
"""Determines if a quota usage refresh is needed.
:param quota_usage: A QuotaUsage object for a given resource.
:param max_age: Number of seconds between subsequent usage refreshes.
:return: True if a refresh is needed, False otherwise.
"""
refresh = False
if quota_usage.in_use < 0:
# Negative in_use count indicates a desync, so try to
# heal from that...
LOG.debug('in_use has dropped below 0; forcing refresh for '
'QuotaUsage: %s', dict(quota_usage))
refresh = True
elif quota_usage.until_refresh is not None:
quota_usage.until_refresh -= 1
if quota_usage.until_refresh <= 0:
refresh = True
elif max_age and (timeutils.utcnow() -
quota_usage.updated_at).seconds >= max_age:
refresh = True
return refresh
def _refresh_quota_usages(quota_usage, until_refresh, in_use):
"""Refreshes quota usage for the given resource.
:param quota_usage: A QuotaUsage object for a given resource.
:param until_refresh: Count of reservations until usage is refreshed,
int or None
:param in_use: Actual quota usage for the resource.
"""
if quota_usage.in_use != in_use:
LOG.info('quota_usages out of sync, updating. '
'project_id: %(project_id)s, '
'user_id: %(user_id)s, '
'resource: %(res)s, '
'tracked usage: %(tracked_use)s, '
'actual usage: %(in_use)s',
{'project_id': quota_usage.project_id,
'user_id': quota_usage.user_id,
'res': quota_usage.resource,
'tracked_use': quota_usage.in_use,
'in_use': in_use})
else:
LOG.debug('QuotaUsage has not changed, refresh is unnecessary for: %s',
dict(quota_usage))
# Update the usage
quota_usage.in_use = in_use
quota_usage.until_refresh = until_refresh or None
def _refresh_quota_usages_if_needed(user_usages, context, resources, keys,
project_id, user_id, until_refresh,
max_age, force_refresh=False):
elevated = context.elevated()
# Handle usage refresh
work = set(keys)
while work:
resource = work.pop()
# Do we need to refresh the usage?
created = _create_quota_usage_if_missing(user_usages, resource,
until_refresh, project_id,
user_id, context.session)
refresh = force_refresh
if not refresh:
refresh = created or \
_is_quota_refresh_needed(user_usages[resource], max_age)
# OK, refresh the usage
if refresh:
# Grab the sync routine
sync = QUOTA_SYNC_FUNCTIONS[resources[resource].sync]
updates = sync(elevated, project_id, user_id)
for res, in_use in updates.items():
# Make sure we have a destination for the usage!
_create_quota_usage_if_missing(user_usages, res,
until_refresh, project_id,
user_id, context.session)
_refresh_quota_usages(user_usages[res], until_refresh,
in_use)
# Because more than one resource may be refreshed
# by the call to the sync routine, and we don't
# want to double-sync, we make sure all refreshed
# resources are dropped from the work set.
work.discard(res)
# NOTE(Vek): We make the assumption that the sync
# routine actually refreshes the
# resources that it is the sync routine
# for. We don't check, because this is
# a best-effort mechanism.
def _calculate_overquota(project_quotas, user_quotas, deltas,
project_usages, user_usages):
"""Checks if any resources will go over quota based on the request.
:param project_quotas: dict of resource quotas (limits) for the project.
:param user_quotas: dict of resource quotas (limits) for the user.
:param deltas: dict of resource keys to positive/negative quota
changes for the resources in a given operation.
:param project_usages: dict of resource keys to QuotaUsage records for the
project.
:param user_usages: dict of resource keys to QuotaUsage records for the
user.
:return: list of resources that are over-quota for the
operation.
"""
overs = []
for res, delta in deltas.items():
# We can't go over-quota if we're not reserving anything.
if delta >= 0:
# We can't go over-quota if we have unlimited quotas.
# over if the project usage + delta is more than project quota
if 0 <= project_quotas[res] < delta + project_usages[res]['total']:
LOG.debug('Request is over project quota for resource '
'"%(res)s". Project limit: %(limit)s, delta: '
'%(delta)s, current total project usage: %(total)s',
{'res': res, 'limit': project_quotas[res],
'delta': delta,
'total': project_usages[res]['total']})
overs.append(res)
# We can't go over-quota if we have unlimited quotas.
# over if the user usage + delta is more than user quota
elif 0 <= user_quotas[res] < delta + user_usages[res]['total']:
LOG.debug('Request is over user quota for resource '
'"%(res)s". User limit: %(limit)s, delta: '
'%(delta)s, current total user usage: %(total)s',
{'res': res, 'limit': user_quotas[res],
'delta': delta, 'total': user_usages[res]['total']})
overs.append(res)
return overs
@require_context
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
@pick_context_manager_writer
def quota_usage_refresh(context, resources, keys, until_refresh, max_age,
project_id=None, user_id=None):
if project_id is None:
project_id = context.project_id
if user_id is None:
user_id = context.user_id
# Get the current usages
project_usages, user_usages = _get_project_user_quota_usages(
context, project_id, user_id)
# Force refresh of the usages
_refresh_quota_usages_if_needed(user_usages, context, resources, keys,
project_id, user_id, until_refresh,
max_age, force_refresh=True)
@require_context
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
@pick_context_manager_writer
def quota_reserve(context, resources, project_quotas, user_quotas, deltas,
expire, until_refresh, max_age, project_id=None,
user_id=None):
if project_id is None:
project_id = context.project_id
if user_id is None:
user_id = context.user_id
# Get the current usages
project_usages, user_usages = _get_project_user_quota_usages(
context, project_id, user_id)
_refresh_quota_usages_if_needed(user_usages, context, resources,
deltas.keys(), project_id, user_id,
until_refresh, max_age)
# Check for deltas that would go negative
unders = [res for res, delta in deltas.items()
if delta < 0 and
delta + user_usages[res].in_use < 0]
# Now, let's check the quotas
# NOTE(Vek): We're only concerned about positive increments.
# If a project has gone over quota, we want them to
# be able to reduce their usage without any
# problems.
for key, value in user_usages.items():
if key not in project_usages:
LOG.debug('Copying QuotaUsage for resource "%(key)s" from '
'user_usages into project_usages: %(value)s',
{'key': key, 'value': dict(value)})
project_usages[key] = value
overs = _calculate_overquota(project_quotas, user_quotas, deltas,
project_usages, user_usages)
# NOTE(Vek): The quota check needs to be in the transaction,
# but the transaction doesn't fail just because
# we're over quota, so the OverQuota raise is
# outside the transaction. If we did the raise
# here, our usage updates would be discarded, but
# they're not invalidated by being over-quota.
# Create the reservations
if not overs:
reservations = []
for res, delta in deltas.items():
reservation = _reservation_create(
uuidutils.generate_uuid(),
user_usages[res],
project_id,
user_id,
res, delta, expire,
context.session)
reservations.append(reservation.uuid)
# Also update the reserved quantity
# NOTE(Vek): Again, we are only concerned here about
# positive increments. Here, though, we're
# worried about the following scenario:
#
# 1) User initiates resize down.
# 2) User allocates a new instance.
# 3) Resize down fails or is reverted.
# 4) User is now over quota.
#
# To prevent this, we only update the
# reserved value if the delta is positive.
if delta > 0:
user_usages[res].reserved += delta
# Apply updates to the usages table
for usage_ref in user_usages.values():
context.session.add(usage_ref)
if unders:
LOG.warning("Change will make usage less than 0 for the following "
"resources: %s", unders)
if overs:
if project_quotas == user_quotas:
usages = project_usages
else:
# NOTE(mriedem): user_usages is a dict of resource keys to
# QuotaUsage sqlalchemy dict-like objects and doesn't log well
# so convert the user_usages values to something useful for
# logging. Remove this if we ever change how
# _get_project_user_quota_usages returns the user_usages values.
user_usages = {k: dict(in_use=v['in_use'], reserved=v['reserved'],
total=v['total'])
for k, v in user_usages.items()}
usages = user_usages
usages = {k: dict(in_use=v['in_use'], reserved=v['reserved'])
for k, v in usages.items()}
LOG.debug('Raise OverQuota exception because: '
'project_quotas: %(project_quotas)s, '
'user_quotas: %(user_quotas)s, deltas: %(deltas)s, '
'overs: %(overs)s, project_usages: %(project_usages)s, '
'user_usages: %(user_usages)s',
{'project_quotas': project_quotas,
'user_quotas': user_quotas,
'overs': overs, 'deltas': deltas,
'project_usages': project_usages,
'user_usages': user_usages})
raise exception.OverQuota(overs=sorted(overs), quotas=user_quotas,
usages=usages)
return reservations
def _quota_reservations_query(context, reservations):
"""Return the relevant reservations."""
# Get the listed reservations
return model_query(context, models.Reservation, read_deleted="no").\
filter(models.Reservation.uuid.in_(reservations)).\
with_lockmode('update')
@require_context
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
@pick_context_manager_writer
def reservation_commit(context, reservations, project_id=None, user_id=None):
_project_usages, user_usages = _get_project_user_quota_usages(
context, project_id, user_id)
reservation_query = _quota_reservations_query(context, reservations)
for reservation in reservation_query.all():
usage = user_usages[reservation.resource]
if reservation.delta >= 0:
usage.reserved -= reservation.delta
usage.in_use += reservation.delta
reservation_query.soft_delete(synchronize_session=False)
@require_context
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
@pick_context_manager_writer
def reservation_rollback(context, reservations, project_id=None, user_id=None):
_project_usages, user_usages = _get_project_user_quota_usages(
context, project_id, user_id)
reservation_query = _quota_reservations_query(context, reservations)
for reservation in reservation_query.all():
usage = user_usages[reservation.resource]
if reservation.delta >= 0:
usage.reserved -= reservation.delta
reservation_query.soft_delete(synchronize_session=False)
@pick_context_manager_writer
def quota_destroy_all_by_project_and_user(context, project_id, user_id):
model_query(context, models.ProjectUserQuota, read_deleted="no").\
@ -4553,20 +4047,6 @@ def _security_group_ensure_default(context):
'user_id': context.user_id,
'project_id': context.project_id}
default_group = security_group_create(context, values)
usage = model_query(context, models.QuotaUsage, read_deleted="no").\
filter_by(project_id=context.project_id).\
filter_by(user_id=context.user_id).\
filter_by(resource='security_groups')
# Create quota usage for auto created default security group
if not usage.first():
_quota_usage_create(context.project_id,
context.user_id,
'security_groups',
1, 0,
CONF.quota.until_refresh,
context.session)
else:
usage.update({'in_use': int(usage.first().in_use) + 1})
default_rules = _security_group_rule_get_default_query(context).all()
for default_rule in default_rules: