Merge "Reservations support"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
f5344dec5b
@ -416,13 +416,15 @@ class Controller(object):
|
||||
if self._collection in body:
|
||||
# Have to account for bulk create
|
||||
items = body[self._collection]
|
||||
deltas = {}
|
||||
bulk = True
|
||||
else:
|
||||
items = [body]
|
||||
bulk = False
|
||||
# Ensure policy engine is initialized
|
||||
policy.init()
|
||||
# Store requested resource amounts grouping them by tenant
|
||||
# This won't work with multiple resources. However because of the
|
||||
# current structure of this controller there will hardly be more than
|
||||
# one resource for which reservations are being made
|
||||
request_deltas = {}
|
||||
for item in items:
|
||||
self._validate_network_tenant_ownership(request,
|
||||
item[self._resource])
|
||||
@ -433,30 +435,34 @@ class Controller(object):
|
||||
if 'tenant_id' not in item[self._resource]:
|
||||
# no tenant_id - no quota check
|
||||
continue
|
||||
try:
|
||||
tenant_id = item[self._resource]['tenant_id']
|
||||
count = quota.QUOTAS.count(request.context, self._resource,
|
||||
self._plugin, tenant_id)
|
||||
if bulk:
|
||||
delta = deltas.get(tenant_id, 0) + 1
|
||||
deltas[tenant_id] = delta
|
||||
else:
|
||||
delta = 1
|
||||
kwargs = {self._resource: count + delta}
|
||||
except exceptions.QuotaResourceUnknown as e:
|
||||
tenant_id = item[self._resource]['tenant_id']
|
||||
delta = request_deltas.get(tenant_id, 0)
|
||||
delta = delta + 1
|
||||
request_deltas[tenant_id] = delta
|
||||
# Quota enforcement
|
||||
reservations = []
|
||||
try:
|
||||
for tenant in request_deltas:
|
||||
reservation = quota.QUOTAS.make_reservation(
|
||||
request.context,
|
||||
tenant,
|
||||
{self._resource:
|
||||
request_deltas[tenant]},
|
||||
self._plugin)
|
||||
reservations.append(reservation)
|
||||
except exceptions.QuotaResourceUnknown as e:
|
||||
# We don't want to quota this resource
|
||||
LOG.debug(e)
|
||||
else:
|
||||
quota.QUOTAS.limit_check(request.context,
|
||||
item[self._resource]['tenant_id'],
|
||||
**kwargs)
|
||||
|
||||
def notify(create_result):
|
||||
# Ensure usage trackers for all resources affected by this API
|
||||
# operation are marked as dirty
|
||||
# TODO(salv-orlando): This operation will happen in a single
|
||||
# transaction with reservation commit once that is implemented
|
||||
resource_registry.set_resources_dirty(request.context)
|
||||
with request.context.session.begin():
|
||||
# Commit the reservation(s)
|
||||
for reservation in reservations:
|
||||
quota.QUOTAS.commit_reservation(
|
||||
request.context, reservation.reservation_id)
|
||||
resource_registry.set_resources_dirty(request.context)
|
||||
|
||||
notifier_method = self._resource + '.create.end'
|
||||
self._notifier.info(request.context,
|
||||
@ -467,11 +473,35 @@ class Controller(object):
|
||||
notifier_method)
|
||||
return create_result
|
||||
|
||||
kwargs = {self._parent_id_name: parent_id} if parent_id else {}
|
||||
def do_create(body, bulk=False, emulated=False):
|
||||
kwargs = {self._parent_id_name: parent_id} if parent_id else {}
|
||||
if bulk and not emulated:
|
||||
obj_creator = getattr(self._plugin, "%s_bulk" % action)
|
||||
else:
|
||||
obj_creator = getattr(self._plugin, action)
|
||||
try:
|
||||
if emulated:
|
||||
return self._emulate_bulk_create(obj_creator, request,
|
||||
body, parent_id)
|
||||
else:
|
||||
if self._collection in body:
|
||||
# This is weird but fixing it requires changes to the
|
||||
# plugin interface
|
||||
kwargs.update({self._collection: body})
|
||||
else:
|
||||
kwargs.update({self._resource: body})
|
||||
return obj_creator(request.context, **kwargs)
|
||||
except Exception:
|
||||
# In case of failure the plugin will always raise an
|
||||
# exception. Cancel the reservation
|
||||
with excutils.save_and_reraise_exception():
|
||||
for reservation in reservations:
|
||||
quota.QUOTAS.cancel_reservation(
|
||||
request.context, reservation.reservation_id)
|
||||
|
||||
if self._collection in body and self._native_bulk:
|
||||
# plugin does atomic bulk create operations
|
||||
obj_creator = getattr(self._plugin, "%s_bulk" % action)
|
||||
objs = obj_creator(request.context, body, **kwargs)
|
||||
objs = do_create(body, bulk=True)
|
||||
# Use first element of list to discriminate attributes which
|
||||
# should be removed because of authZ policies
|
||||
fields_to_strip = self._exclude_attributes_by_policy(
|
||||
@ -480,15 +510,12 @@ class Controller(object):
|
||||
request.context, obj, fields_to_strip=fields_to_strip)
|
||||
for obj in objs]})
|
||||
else:
|
||||
obj_creator = getattr(self._plugin, action)
|
||||
if self._collection in body:
|
||||
# Emulate atomic bulk behavior
|
||||
objs = self._emulate_bulk_create(obj_creator, request,
|
||||
body, parent_id)
|
||||
objs = do_create(body, bulk=True, emulated=True)
|
||||
return notify({self._collection: objs})
|
||||
else:
|
||||
kwargs.update({self._resource: body})
|
||||
obj = obj_creator(request.context, **kwargs)
|
||||
obj = do_create(body)
|
||||
self._send_nova_notification(action, {},
|
||||
{self._resource: obj})
|
||||
return notify({self._resource: self._view(request.context,
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
2a16083502f3
|
||||
48153cb5f051
|
||||
9859ac9c136
|
||||
kilo
|
||||
|
||||
@ -0,0 +1,47 @@
|
||||
# Copyright 2015 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
|
||||
"""quota_reservations
|
||||
|
||||
Revision ID: 9859ac9c136
|
||||
Revises: 48153cb5f051
|
||||
Create Date: 2015-03-11 06:40:56.775075
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '9859ac9c136'
|
||||
down_revision = '48153cb5f051'
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
'reservations',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=255), nullable=True),
|
||||
sa.Column('expiration', sa.DateTime(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'))
|
||||
|
||||
op.create_table(
|
||||
'resourcedeltas',
|
||||
sa.Column('resource', sa.String(length=255), nullable=False),
|
||||
sa.Column('reservation_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('amount', sa.Integer(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['reservation_id'], ['reservations.id'],
|
||||
ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('resource', 'reservation_id'))
|
||||
@ -13,11 +13,21 @@
|
||||
# under the License.
|
||||
|
||||
import collections
|
||||
import datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import exc as orm_exc
|
||||
from sqlalchemy import sql
|
||||
|
||||
from neutron.db import common_db_mixin as common_db_api
|
||||
from neutron.db.quota import models as quota_models
|
||||
|
||||
|
||||
# Wrapper for utcnow - needed for mocking it in unit tests
|
||||
def utcnow():
|
||||
return datetime.datetime.utcnow()
|
||||
|
||||
|
||||
class QuotaUsageInfo(collections.namedtuple(
|
||||
'QuotaUsageInfo', ['resource', 'tenant_id', 'used', 'reserved', 'dirty'])):
|
||||
|
||||
@ -27,6 +37,32 @@ class QuotaUsageInfo(collections.namedtuple(
|
||||
return self.reserved + self.used
|
||||
|
||||
|
||||
class ReservationInfo(object):
|
||||
"""Information about a resource reservation."""
|
||||
|
||||
def __init__(self, reservation_id, tenant_id, expiration, deltas):
|
||||
self._reservation_id = reservation_id
|
||||
self._tenant_id = tenant_id
|
||||
self._expiration = expiration
|
||||
self._deltas = deltas
|
||||
|
||||
@property
|
||||
def reservation_id(self):
|
||||
return self._reservation_id
|
||||
|
||||
@property
|
||||
def tenant_id(self):
|
||||
return self._tenant_id
|
||||
|
||||
@property
|
||||
def expiration(self):
|
||||
return self._expiration
|
||||
|
||||
@property
|
||||
def deltas(self):
|
||||
return self._deltas
|
||||
|
||||
|
||||
def get_quota_usage_by_resource_and_tenant(context, resource, tenant_id,
|
||||
lock_for_update=False):
|
||||
"""Return usage info for a given resource and tenant.
|
||||
@ -157,3 +193,105 @@ def set_all_quota_usage_dirty(context, resource, dirty=True):
|
||||
query = common_db_api.model_query(context, quota_models.QuotaUsage)
|
||||
query = query.filter_by(resource=resource)
|
||||
return query.update({'dirty': dirty})
|
||||
|
||||
|
||||
def create_reservation(context, tenant_id, deltas, expiration=None):
|
||||
# This method is usually called from within another transaction.
|
||||
# Consider using begin_nested
|
||||
with context.session.begin(subtransactions=True):
|
||||
expiration = expiration or (utcnow() + datetime.timedelta(0, 120))
|
||||
resv = quota_models.Reservation(tenant_id=tenant_id,
|
||||
expiration=expiration)
|
||||
context.session.add(resv)
|
||||
for (resource, delta) in deltas.items():
|
||||
context.session.add(
|
||||
quota_models.ResourceDelta(resource=resource,
|
||||
amount=delta,
|
||||
reservation=resv))
|
||||
# quota_usage for all resources involved in this reservation must
|
||||
# be marked as dirty
|
||||
set_resources_quota_usage_dirty(
|
||||
context, deltas.keys(), tenant_id)
|
||||
return ReservationInfo(resv['id'],
|
||||
resv['tenant_id'],
|
||||
resv['expiration'],
|
||||
dict((delta.resource, delta.amount)
|
||||
for delta in resv.resource_deltas))
|
||||
|
||||
|
||||
def get_reservation(context, reservation_id):
|
||||
query = context.session.query(quota_models.Reservation).filter_by(
|
||||
id=reservation_id)
|
||||
resv = query.first()
|
||||
if not resv:
|
||||
return
|
||||
return ReservationInfo(resv['id'],
|
||||
resv['tenant_id'],
|
||||
resv['expiration'],
|
||||
dict((delta.resource, delta.amount)
|
||||
for delta in resv.resource_deltas))
|
||||
|
||||
|
||||
def remove_reservation(context, reservation_id, set_dirty=False):
|
||||
delete_query = context.session.query(quota_models.Reservation).filter_by(
|
||||
id=reservation_id)
|
||||
# Not handling MultipleResultsFound as the query is filtering by primary
|
||||
# key
|
||||
try:
|
||||
reservation = delete_query.one()
|
||||
except orm_exc.NoResultFound:
|
||||
# TODO(salv-orlando): Raise here and then handle the exception?
|
||||
return
|
||||
tenant_id = reservation.tenant_id
|
||||
resources = [delta.resource for delta in reservation.resource_deltas]
|
||||
num_deleted = delete_query.delete()
|
||||
if set_dirty:
|
||||
# quota_usage for all resource involved in this reservation must
|
||||
# be marked as dirty
|
||||
set_resources_quota_usage_dirty(context, resources, tenant_id)
|
||||
return num_deleted
|
||||
|
||||
|
||||
def get_reservations_for_resources(context, tenant_id, resources,
|
||||
expired=False):
|
||||
"""Retrieve total amount of reservations for specified resources.
|
||||
|
||||
:param context: Neutron context with db session
|
||||
:param tenant_id: Tenant identifier
|
||||
:param resources: Resources for which reserved amounts should be fetched
|
||||
:param expired: False to fetch active reservations, True to fetch expired
|
||||
reservations (defaults to False)
|
||||
:returns: a dictionary mapping resources with corresponding deltas
|
||||
"""
|
||||
if not resources:
|
||||
# Do not waste time
|
||||
return
|
||||
now = utcnow()
|
||||
resv_query = context.session.query(
|
||||
quota_models.ResourceDelta.resource,
|
||||
quota_models.Reservation.expiration,
|
||||
sql.func.sum(quota_models.ResourceDelta.amount)).join(
|
||||
quota_models.Reservation)
|
||||
if expired:
|
||||
exp_expr = (quota_models.Reservation.expiration < now)
|
||||
else:
|
||||
exp_expr = (quota_models.Reservation.expiration >= now)
|
||||
resv_query = resv_query.filter(sa.and_(
|
||||
quota_models.Reservation.tenant_id == tenant_id,
|
||||
quota_models.ResourceDelta.resource.in_(resources),
|
||||
exp_expr)).group_by(
|
||||
quota_models.ResourceDelta.resource)
|
||||
return dict((resource, total_reserved)
|
||||
for (resource, exp, total_reserved) in resv_query)
|
||||
|
||||
|
||||
def remove_expired_reservations(context, tenant_id=None):
|
||||
now = utcnow()
|
||||
resv_query = context.session.query(quota_models.Reservation)
|
||||
if tenant_id:
|
||||
tenant_expr = (quota_models.Reservation.tenant_id == tenant_id)
|
||||
else:
|
||||
tenant_expr = sql.true()
|
||||
resv_query = resv_query.filter(sa.and_(
|
||||
tenant_expr, quota_models.Reservation.expiration < now))
|
||||
return resv_query.delete()
|
||||
|
||||
@ -13,9 +13,16 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from oslo_db import api as oslo_db_api
|
||||
from oslo_log import log
|
||||
|
||||
from neutron.common import exceptions
|
||||
from neutron.db import api as db_api
|
||||
from neutron.db.quota import api as quota_api
|
||||
from neutron.db.quota import models as quota_models
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
class DbQuotaDriver(object):
|
||||
"""Driver to perform necessary checks to enforce quotas and obtain quota
|
||||
@ -42,7 +49,8 @@ class DbQuotaDriver(object):
|
||||
# update with tenant specific limits
|
||||
q_qry = context.session.query(quota_models.Quota).filter_by(
|
||||
tenant_id=tenant_id)
|
||||
tenant_quota.update((q['resource'], q['limit']) for q in q_qry)
|
||||
for item in q_qry:
|
||||
tenant_quota[item['resource']] = item['limit']
|
||||
|
||||
return tenant_quota
|
||||
|
||||
@ -116,6 +124,112 @@ class DbQuotaDriver(object):
|
||||
|
||||
return dict((k, v) for k, v in quotas.items())
|
||||
|
||||
def _handle_expired_reservations(self, context, tenant_id,
|
||||
resource, expired_amount):
|
||||
LOG.debug(("Adjusting usage for resource %(resource)s: "
|
||||
"removing %(expired)d reserved items"),
|
||||
{'resource': resource,
|
||||
'expired': expired_amount})
|
||||
# TODO(salv-orlando): It should be possible to do this
|
||||
# operation for all resources with a single query.
|
||||
# Update reservation usage
|
||||
quota_api.set_quota_usage(
|
||||
context,
|
||||
resource,
|
||||
tenant_id,
|
||||
reserved=-expired_amount,
|
||||
delta=True)
|
||||
# Delete expired reservations (we don't want them to accrue
|
||||
# in the database)
|
||||
quota_api.remove_expired_reservations(
|
||||
context, tenant_id=tenant_id)
|
||||
|
||||
@oslo_db_api.wrap_db_retry(max_retries=db_api.MAX_RETRIES,
|
||||
retry_on_request=True,
|
||||
retry_on_deadlock=True)
|
||||
def make_reservation(self, context, tenant_id, resources, deltas, plugin):
|
||||
# Lock current reservation table
|
||||
# NOTE(salv-orlando): This routine uses DB write locks.
|
||||
# These locks are acquired by the count() method invoked on resources.
|
||||
# Please put your shotguns aside.
|
||||
# A non locking algorithm for handling reservation is feasible, however
|
||||
# it will require two database writes even in cases when there are not
|
||||
# concurrent reservations.
|
||||
# For this reason it might be advisable to handle contention using
|
||||
# this kind of locks and paying the cost of a write set certification
|
||||
# failure when a mysql galera cluster is employed. Also, this class of
|
||||
# locks should be ok to use when support for sending "hotspot" writes
|
||||
# to a single node will be avaialable.
|
||||
requested_resources = deltas.keys()
|
||||
with context.session.begin():
|
||||
# Gather current usage information
|
||||
# TODO(salv-orlando): calling count() for every resource triggers
|
||||
# multiple queries on quota usage. This should be improved, however
|
||||
# this is not an urgent matter as the REST API currently only
|
||||
# allows allocation of a resource at a time
|
||||
# NOTE: pass plugin too for compatibility with CountableResource
|
||||
# instances
|
||||
current_usages = dict(
|
||||
(resource, resources[resource].count(
|
||||
context, plugin, tenant_id)) for
|
||||
resource in requested_resources)
|
||||
# get_tenant_quotes needs in inout a dictionary mapping resource
|
||||
# name to BaseResosurce instances so that the default quota can be
|
||||
# retrieved
|
||||
current_limits = self.get_tenant_quotas(
|
||||
context, resources, tenant_id)
|
||||
# Adjust for expired reservations. Apparently it is cheaper than
|
||||
# querying everytime for active reservations and counting overall
|
||||
# quantity of resources reserved
|
||||
expired_deltas = quota_api.get_reservations_for_resources(
|
||||
context, tenant_id, requested_resources, expired=True)
|
||||
# Verify that the request can be accepted with current limits
|
||||
resources_over_limit = []
|
||||
for resource in requested_resources:
|
||||
expired_reservations = expired_deltas.get(resource, 0)
|
||||
total_usage = current_usages[resource] - expired_reservations
|
||||
# A negative quota limit means infinite
|
||||
if current_limits[resource] < 0:
|
||||
LOG.debug(("Resource %(resource)s has unlimited quota "
|
||||
"limit. It is possible to allocate %(delta)s "
|
||||
"items."), {'resource': resource,
|
||||
'delta': deltas[resource]})
|
||||
continue
|
||||
res_headroom = current_limits[resource] - total_usage
|
||||
LOG.debug(("Attempting to reserve %(delta)d items for "
|
||||
"resource %(resource)s. Total usage: %(total)d; "
|
||||
"quota limit: %(limit)d; headroom:%(headroom)d"),
|
||||
{'resource': resource,
|
||||
'delta': deltas[resource],
|
||||
'total': total_usage,
|
||||
'limit': current_limits[resource],
|
||||
'headroom': res_headroom})
|
||||
if res_headroom < deltas[resource]:
|
||||
resources_over_limit.append(resource)
|
||||
if expired_reservations:
|
||||
self._handle_expired_reservations(
|
||||
context, tenant_id, resource, expired_reservations)
|
||||
|
||||
if resources_over_limit:
|
||||
raise exceptions.OverQuota(overs=sorted(resources_over_limit))
|
||||
# Success, store the reservation
|
||||
# TODO(salv-orlando): Make expiration time configurable
|
||||
return quota_api.create_reservation(
|
||||
context, tenant_id, deltas)
|
||||
|
||||
def commit_reservation(self, context, reservation_id):
|
||||
# Do not mark resource usage as dirty. If a reservation is committed,
|
||||
# then the releveant resources have been created. Usage data for these
|
||||
# resources has therefore already been marked dirty.
|
||||
quota_api.remove_reservation(context, reservation_id,
|
||||
set_dirty=False)
|
||||
|
||||
def cancel_reservation(self, context, reservation_id):
|
||||
# Mark resource usage as dirty so the next time both actual resources
|
||||
# used and reserved will be recalculated
|
||||
quota_api.remove_reservation(context, reservation_id,
|
||||
set_dirty=True)
|
||||
|
||||
def limit_check(self, context, tenant_id, resources, values):
|
||||
"""Check simple quota limits.
|
||||
|
||||
|
||||
@ -13,12 +13,33 @@
|
||||
# under the License.
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import orm
|
||||
from sqlalchemy import sql
|
||||
|
||||
from neutron.db import model_base
|
||||
from neutron.db import models_v2
|
||||
|
||||
|
||||
class ResourceDelta(model_base.BASEV2):
|
||||
resource = sa.Column(sa.String(255), primary_key=True)
|
||||
reservation_id = sa.Column(sa.String(36),
|
||||
sa.ForeignKey('reservations.id',
|
||||
ondelete='CASCADE'),
|
||||
primary_key=True,
|
||||
nullable=False)
|
||||
# Requested amount of resource
|
||||
amount = sa.Column(sa.Integer)
|
||||
|
||||
|
||||
class Reservation(model_base.BASEV2, models_v2.HasId):
|
||||
tenant_id = sa.Column(sa.String(255))
|
||||
expiration = sa.Column(sa.DateTime())
|
||||
resource_deltas = orm.relationship(ResourceDelta,
|
||||
backref='reservation',
|
||||
lazy="joined",
|
||||
cascade='all, delete-orphan')
|
||||
|
||||
|
||||
class Quota(model_base.BASEV2, models_v2.HasId, models_v2.HasTenant):
|
||||
"""Represent a single quota override for a tenant.
|
||||
|
||||
|
||||
@ -24,6 +24,7 @@ import six
|
||||
import webob
|
||||
|
||||
from neutron.common import exceptions
|
||||
from neutron.db.quota import api as quota_api
|
||||
from neutron.i18n import _LI, _LW
|
||||
from neutron.quota import resource_registry
|
||||
|
||||
@ -152,6 +153,33 @@ class ConfDriver(object):
|
||||
msg = _('Access to this resource was denied.')
|
||||
raise webob.exc.HTTPForbidden(msg)
|
||||
|
||||
def make_reservation(self, context, tenant_id, resources, deltas, plugin):
|
||||
"""This driver does not support reservations.
|
||||
|
||||
This routine is provided for backward compatibility purposes with
|
||||
the API controllers which have now been adapted to make reservations
|
||||
rather than counting resources and checking limits - as this
|
||||
routine ultimately does.
|
||||
"""
|
||||
for resource in deltas.keys():
|
||||
count = QUOTAS.count(context, resource, plugin, tenant_id)
|
||||
total_use = deltas.get(resource, 0) + count
|
||||
deltas[resource] = total_use
|
||||
|
||||
self.limit_check(
|
||||
context,
|
||||
tenant_id,
|
||||
resource_registry.get_all_resources(),
|
||||
deltas)
|
||||
# return a fake reservation - the REST controller expects it
|
||||
return quota_api.ReservationInfo('fake', None, None, None)
|
||||
|
||||
def commit_reservation(self, context, reservation_id):
|
||||
"""Tnis is a noop as this driver does not support reservations."""
|
||||
|
||||
def cancel_reservation(self, context, reservation_id):
|
||||
"""Tnis is a noop as this driver does not support reservations."""
|
||||
|
||||
|
||||
class QuotaEngine(object):
|
||||
"""Represent the set of recognized quotas."""
|
||||
@ -210,6 +238,39 @@ class QuotaEngine(object):
|
||||
|
||||
return res.count(context, *args, **kwargs)
|
||||
|
||||
def make_reservation(self, context, tenant_id, deltas, plugin):
|
||||
# Verify that resources are managed by the quota engine
|
||||
# Ensure no value is less than zero
|
||||
unders = [key for key, val in deltas.items() if val < 0]
|
||||
if unders:
|
||||
raise exceptions.InvalidQuotaValue(unders=sorted(unders))
|
||||
|
||||
requested_resources = set(deltas.keys())
|
||||
all_resources = resource_registry.get_all_resources()
|
||||
managed_resources = set([res for res in all_resources.keys()
|
||||
if res in requested_resources])
|
||||
# Make sure we accounted for all of them...
|
||||
unknown_resources = requested_resources - managed_resources
|
||||
|
||||
if unknown_resources:
|
||||
raise exceptions.QuotaResourceUnknown(
|
||||
unknown=sorted(unknown_resources))
|
||||
# FIXME(salv-orlando): There should be no reason for sending all the
|
||||
# resource in the registry to the quota driver, but as other driver
|
||||
# APIs request them, this will be sorted out with a different patch.
|
||||
return self.get_driver().make_reservation(
|
||||
context,
|
||||
tenant_id,
|
||||
all_resources,
|
||||
deltas,
|
||||
plugin)
|
||||
|
||||
def commit_reservation(self, context, reservation_id):
|
||||
self.get_driver().commit_reservation(context, reservation_id)
|
||||
|
||||
def cancel_reservation(self, context, reservation_id):
|
||||
self.get_driver().cancel_reservation(context, reservation_id)
|
||||
|
||||
def limit_check(self, context, tenant_id, **values):
|
||||
"""Check simple quota limits.
|
||||
|
||||
@ -232,6 +293,7 @@ class QuotaEngine(object):
|
||||
:param tenant_id: Tenant for which the quota limit is being checked
|
||||
:param values: Dict specifying requested deltas for each resource
|
||||
"""
|
||||
# TODO(salv-orlando): Deprecate calls to this API
|
||||
# Verify that resources are managed by the quota engine
|
||||
requested_resources = set(values.keys())
|
||||
managed_resources = set([res for res in
|
||||
|
||||
@ -208,14 +208,15 @@ class TrackedResource(BaseResource):
|
||||
max_retries=db_api.MAX_RETRIES,
|
||||
exception_checker=lambda exc:
|
||||
isinstance(exc, oslo_db_exception.DBDuplicateEntry))
|
||||
def _set_quota_usage(self, context, tenant_id, in_use):
|
||||
return quota_api.set_quota_usage(context, self.name,
|
||||
tenant_id, in_use=in_use)
|
||||
def _set_quota_usage(self, context, tenant_id, in_use, reserved):
|
||||
return quota_api.set_quota_usage(context, self.name, tenant_id,
|
||||
in_use=in_use, reserved=reserved)
|
||||
|
||||
def _resync(self, context, tenant_id, in_use):
|
||||
def _resync(self, context, tenant_id, in_use, reserved):
|
||||
# Update quota usage
|
||||
usage_info = self._set_quota_usage(
|
||||
context, tenant_id, in_use=in_use)
|
||||
context, tenant_id, in_use, reserved)
|
||||
|
||||
self._dirty_tenants.discard(tenant_id)
|
||||
self._out_of_sync_tenants.discard(tenant_id)
|
||||
LOG.debug(("Unset dirty status for tenant:%(tenant_id)s on "
|
||||
@ -231,40 +232,62 @@ class TrackedResource(BaseResource):
|
||||
{'tenant_id': tenant_id, 'resource': self.name})
|
||||
in_use = context.session.query(self._model_class).filter_by(
|
||||
tenant_id=tenant_id).count()
|
||||
reservations = quota_api.get_reservations_for_resources(
|
||||
context, tenant_id, [self.name])
|
||||
reserved = reservations.get(self.name, 0)
|
||||
# Update quota usage
|
||||
return self._resync(context, tenant_id, in_use)
|
||||
return self._resync(context, tenant_id, in_use, reserved)
|
||||
|
||||
def count(self, context, _plugin, tenant_id, resync_usage=False):
|
||||
"""Return the current usage count for the resource."""
|
||||
# Load current usage data
|
||||
"""Return the current usage count for the resource.
|
||||
|
||||
This method will fetch the information from resource usage data,
|
||||
unless usage data are marked as "dirty", in which case both used and
|
||||
reserved resource are explicitly counted.
|
||||
|
||||
The _plugin and _resource parameters are unused but kept for
|
||||
compatibility with the signature of the count method for
|
||||
CountableResource instances.
|
||||
"""
|
||||
# Load current usage data, setting a row-level lock on the DB
|
||||
usage_info = quota_api.get_quota_usage_by_resource_and_tenant(
|
||||
context, self.name, tenant_id)
|
||||
context, self.name, tenant_id, lock_for_update=True)
|
||||
# If dirty or missing, calculate actual resource usage querying
|
||||
# the database and set/create usage info data
|
||||
# NOTE: this routine "trusts" usage counters at service startup. This
|
||||
# assumption is generally valid, but if the database is tampered with,
|
||||
# or if data migrations do not take care of usage counters, the
|
||||
# assumption will not hold anymore
|
||||
if (tenant_id in self._dirty_tenants or not usage_info
|
||||
or usage_info.dirty):
|
||||
if (tenant_id in self._dirty_tenants or
|
||||
not usage_info or usage_info.dirty):
|
||||
LOG.debug(("Usage tracker for resource:%(resource)s and tenant:"
|
||||
"%(tenant_id)s is out of sync, need to count used "
|
||||
"quota"), {'resource': self.name,
|
||||
'tenant_id': tenant_id})
|
||||
in_use = context.session.query(self._model_class).filter_by(
|
||||
tenant_id=tenant_id).count()
|
||||
reservations = quota_api.get_reservations_for_resources(
|
||||
context, tenant_id, [self.name])
|
||||
reserved = reservations.get(self.name, 0)
|
||||
|
||||
# Update quota usage, if requested (by default do not do that, as
|
||||
# typically one counts before adding a record, and that would mark
|
||||
# the usage counter as dirty again)
|
||||
if resync_usage or not usage_info:
|
||||
usage_info = self._resync(context, tenant_id, in_use)
|
||||
usage_info = self._resync(context, tenant_id,
|
||||
in_use, reserved)
|
||||
else:
|
||||
usage_info = quota_api.QuotaUsageInfo(usage_info.resource,
|
||||
usage_info.tenant_id,
|
||||
in_use,
|
||||
usage_info.reserved,
|
||||
reserved,
|
||||
usage_info.dirty)
|
||||
|
||||
LOG.debug(("Quota usage for %(resource)s was recalculated. "
|
||||
"Used quota:%(used)d; Reserved quota:%(reserved)d"),
|
||||
{'resource': self.name,
|
||||
'used': usage_info.used,
|
||||
'reserved': usage_info.reserved})
|
||||
return usage_info.total
|
||||
|
||||
def register_events(self):
|
||||
|
||||
@ -65,7 +65,7 @@ def set_resources_dirty(context):
|
||||
return
|
||||
|
||||
for res in get_all_resources().values():
|
||||
with context.session.begin():
|
||||
with context.session.begin(subtransactions=True):
|
||||
if is_tracked(res.name) and res.dirty:
|
||||
res.mark_dirty(context, nested=True)
|
||||
|
||||
|
||||
@ -12,6 +12,10 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import datetime
|
||||
|
||||
import mock
|
||||
|
||||
from neutron import context
|
||||
from neutron.db.quota import api as quota_api
|
||||
from neutron.tests.unit import testlib_api
|
||||
@ -24,6 +28,12 @@ class TestQuotaDbApi(testlib_api.SqlTestCaseLight):
|
||||
self.context = context.Context('Gonzalo', self.tenant_id,
|
||||
is_admin=False, is_advsvc=False)
|
||||
|
||||
def _create_reservation(self, resource_deltas,
|
||||
tenant_id=None, expiration=None):
|
||||
tenant_id = tenant_id or self.tenant_id
|
||||
return quota_api.create_reservation(
|
||||
self.context, tenant_id, resource_deltas, expiration)
|
||||
|
||||
def _create_quota_usage(self, resource, used, reserved, tenant_id=None):
|
||||
tenant_id = tenant_id or self.tenant_id
|
||||
return quota_api.set_quota_usage(
|
||||
@ -203,6 +213,125 @@ class TestQuotaDbApi(testlib_api.SqlTestCaseLight):
|
||||
self.assertIsNone(quota_api.get_quota_usage_by_resource_and_tenant(
|
||||
self.context, 'goals', self.tenant_id))
|
||||
|
||||
def _verify_reserved_resources(self, expected, actual):
|
||||
for (resource, delta) in actual.items():
|
||||
self.assertIn(resource, expected)
|
||||
self.assertEqual(delta, expected[resource])
|
||||
del expected[resource]
|
||||
self.assertFalse(expected)
|
||||
|
||||
def test_create_reservation(self):
|
||||
resources = {'goals': 2, 'assists': 1}
|
||||
resv = self._create_reservation(resources)
|
||||
self.assertEqual(self.tenant_id, resv.tenant_id)
|
||||
self._verify_reserved_resources(resources, resv.deltas)
|
||||
|
||||
def test_create_reservation_with_expirtion(self):
|
||||
resources = {'goals': 2, 'assists': 1}
|
||||
exp_date = datetime.datetime(2016, 3, 31, 14, 30)
|
||||
resv = self._create_reservation(resources, expiration=exp_date)
|
||||
self.assertEqual(self.tenant_id, resv.tenant_id)
|
||||
self.assertEqual(exp_date, resv.expiration)
|
||||
self._verify_reserved_resources(resources, resv.deltas)
|
||||
|
||||
def _test_remove_reservation(self, set_dirty):
|
||||
resources = {'goals': 2, 'assists': 1}
|
||||
resv = self._create_reservation(resources)
|
||||
self.assertEqual(1, quota_api.remove_reservation(
|
||||
self.context, resv.reservation_id, set_dirty=set_dirty))
|
||||
|
||||
def test_remove_reservation(self):
|
||||
self._test_remove_reservation(False)
|
||||
|
||||
def test_remove_reservation_and_set_dirty(self):
|
||||
routine = 'neutron.db.quota.api.set_resources_quota_usage_dirty'
|
||||
with mock.patch(routine) as mock_routine:
|
||||
self._test_remove_reservation(False)
|
||||
mock_routine.assert_called_once_with(
|
||||
self.context, mock.ANY, self.tenant_id)
|
||||
|
||||
def test_remove_non_existent_reservation(self):
|
||||
self.assertIsNone(quota_api.remove_reservation(self.context, 'meh'))
|
||||
|
||||
def _get_reservations_for_resource_helper(self):
|
||||
# create three reservation, 1 expired
|
||||
resources_1 = {'goals': 2, 'assists': 1}
|
||||
resources_2 = {'goals': 3, 'bookings': 1}
|
||||
resources_3 = {'bookings': 2, 'assists': 2}
|
||||
exp_date_1 = datetime.datetime(2016, 3, 31, 14, 30)
|
||||
exp_date_2 = datetime.datetime(2015, 3, 31, 14, 30)
|
||||
self._create_reservation(resources_1, expiration=exp_date_1)
|
||||
self._create_reservation(resources_2, expiration=exp_date_1)
|
||||
self._create_reservation(resources_3, expiration=exp_date_2)
|
||||
|
||||
def test_get_reservations_for_resources(self):
|
||||
with mock.patch('neutron.db.quota.api.utcnow') as mock_utcnow:
|
||||
self._get_reservations_for_resource_helper()
|
||||
mock_utcnow.return_value = datetime.datetime(
|
||||
2015, 5, 20, 0, 0)
|
||||
deltas = quota_api.get_reservations_for_resources(
|
||||
self.context, self.tenant_id, ['goals', 'assists', 'bookings'])
|
||||
self.assertIn('goals', deltas)
|
||||
self.assertEqual(5, deltas['goals'])
|
||||
self.assertIn('assists', deltas)
|
||||
self.assertEqual(1, deltas['assists'])
|
||||
self.assertIn('bookings', deltas)
|
||||
self.assertEqual(1, deltas['bookings'])
|
||||
self.assertEqual(3, len(deltas))
|
||||
|
||||
def test_get_expired_reservations_for_resources(self):
|
||||
with mock.patch('neutron.db.quota.api.utcnow') as mock_utcnow:
|
||||
mock_utcnow.return_value = datetime.datetime(
|
||||
2015, 5, 20, 0, 0)
|
||||
self._get_reservations_for_resource_helper()
|
||||
deltas = quota_api.get_reservations_for_resources(
|
||||
self.context, self.tenant_id,
|
||||
['goals', 'assists', 'bookings'],
|
||||
expired=True)
|
||||
self.assertIn('assists', deltas)
|
||||
self.assertEqual(2, deltas['assists'])
|
||||
self.assertIn('bookings', deltas)
|
||||
self.assertEqual(2, deltas['bookings'])
|
||||
self.assertEqual(2, len(deltas))
|
||||
|
||||
def test_get_reservation_for_resources_with_empty_list(self):
|
||||
self.assertIsNone(quota_api.get_reservations_for_resources(
|
||||
self.context, self.tenant_id, []))
|
||||
|
||||
def test_remove_expired_reservations(self):
|
||||
with mock.patch('neutron.db.quota.api.utcnow') as mock_utcnow:
|
||||
mock_utcnow.return_value = datetime.datetime(
|
||||
2015, 5, 20, 0, 0)
|
||||
resources = {'goals': 2, 'assists': 1}
|
||||
exp_date_1 = datetime.datetime(2016, 3, 31, 14, 30)
|
||||
resv_1 = self._create_reservation(resources, expiration=exp_date_1)
|
||||
exp_date_2 = datetime.datetime(2015, 3, 31, 14, 30)
|
||||
resv_2 = self._create_reservation(resources, expiration=exp_date_2)
|
||||
self.assertEqual(1, quota_api.remove_expired_reservations(
|
||||
self.context, self.tenant_id))
|
||||
self.assertIsNone(quota_api.get_reservation(
|
||||
self.context, resv_2.reservation_id))
|
||||
self.assertIsNotNone(quota_api.get_reservation(
|
||||
self.context, resv_1.reservation_id))
|
||||
|
||||
def test_remove_expired_reservations_no_tenant(self):
|
||||
with mock.patch('neutron.db.quota.api.utcnow') as mock_utcnow:
|
||||
mock_utcnow.return_value = datetime.datetime(
|
||||
2015, 5, 20, 0, 0)
|
||||
resources = {'goals': 2, 'assists': 1}
|
||||
exp_date_1 = datetime.datetime(2014, 3, 31, 14, 30)
|
||||
resv_1 = self._create_reservation(resources, expiration=exp_date_1)
|
||||
exp_date_2 = datetime.datetime(2015, 3, 31, 14, 30)
|
||||
resv_2 = self._create_reservation(resources,
|
||||
expiration=exp_date_2,
|
||||
tenant_id='Callejon')
|
||||
self.assertEqual(2, quota_api.remove_expired_reservations(
|
||||
self.context))
|
||||
self.assertIsNone(quota_api.get_reservation(
|
||||
self.context, resv_2.reservation_id))
|
||||
self.assertIsNone(quota_api.get_reservation(
|
||||
self.context, resv_1.reservation_id))
|
||||
|
||||
|
||||
class TestQuotaDbApiAdminContext(TestQuotaDbApi):
|
||||
|
||||
|
||||
@ -27,16 +27,22 @@ class FakePlugin(base_plugin.NeutronDbPluginV2, driver.DbQuotaDriver):
|
||||
class TestResource(object):
|
||||
"""Describe a test resource for quota checking."""
|
||||
|
||||
def __init__(self, name, default):
|
||||
def __init__(self, name, default, fake_count=0):
|
||||
self.name = name
|
||||
self.quota = default
|
||||
self.fake_count = fake_count
|
||||
|
||||
@property
|
||||
def default(self):
|
||||
return self.quota
|
||||
|
||||
def count(self, *args, **kwargs):
|
||||
return self.fake_count
|
||||
|
||||
|
||||
PROJECT = 'prj_test'
|
||||
RESOURCE = 'res_test'
|
||||
ALT_RESOURCE = 'res_test_meh'
|
||||
|
||||
|
||||
class TestDbQuotaDriver(testlib_api.SqlTestCase):
|
||||
@ -132,3 +138,63 @@ class TestDbQuotaDriver(testlib_api.SqlTestCase):
|
||||
self.assertRaises(exceptions.InvalidQuotaValue,
|
||||
self.plugin.limit_check, context.get_admin_context(),
|
||||
PROJECT, resources, values)
|
||||
|
||||
def _test_make_reservation_success(self, quota_driver,
|
||||
resource_name, deltas):
|
||||
resources = {resource_name: TestResource(resource_name, 2)}
|
||||
self.plugin.update_quota_limit(self.context, PROJECT, resource_name, 2)
|
||||
reservation = quota_driver.make_reservation(
|
||||
self.context,
|
||||
self.context.tenant_id,
|
||||
resources,
|
||||
deltas,
|
||||
self.plugin)
|
||||
self.assertIn(resource_name, reservation.deltas)
|
||||
self.assertEqual(deltas[resource_name],
|
||||
reservation.deltas[resource_name])
|
||||
self.assertEqual(self.context.tenant_id,
|
||||
reservation.tenant_id)
|
||||
|
||||
def test_make_reservation_single_resource(self):
|
||||
quota_driver = driver.DbQuotaDriver()
|
||||
self._test_make_reservation_success(
|
||||
quota_driver, RESOURCE, {RESOURCE: 1})
|
||||
|
||||
def test_make_reservation_fill_quota(self):
|
||||
quota_driver = driver.DbQuotaDriver()
|
||||
self._test_make_reservation_success(
|
||||
quota_driver, RESOURCE, {RESOURCE: 2})
|
||||
|
||||
def test_make_reservation_multiple_resources(self):
|
||||
quota_driver = driver.DbQuotaDriver()
|
||||
resources = {RESOURCE: TestResource(RESOURCE, 2),
|
||||
ALT_RESOURCE: TestResource(ALT_RESOURCE, 2)}
|
||||
deltas = {RESOURCE: 1, ALT_RESOURCE: 2}
|
||||
self.plugin.update_quota_limit(self.context, PROJECT, RESOURCE, 2)
|
||||
self.plugin.update_quota_limit(self.context, PROJECT, ALT_RESOURCE, 2)
|
||||
reservation = quota_driver.make_reservation(
|
||||
self.context,
|
||||
self.context.tenant_id,
|
||||
resources,
|
||||
deltas,
|
||||
self.plugin)
|
||||
self.assertIn(RESOURCE, reservation.deltas)
|
||||
self.assertIn(ALT_RESOURCE, reservation.deltas)
|
||||
self.assertEqual(1, reservation.deltas[RESOURCE])
|
||||
self.assertEqual(2, reservation.deltas[ALT_RESOURCE])
|
||||
self.assertEqual(self.context.tenant_id,
|
||||
reservation.tenant_id)
|
||||
|
||||
def test_make_reservation_over_quota_fails(self):
|
||||
quota_driver = driver.DbQuotaDriver()
|
||||
resources = {RESOURCE: TestResource(RESOURCE, 2,
|
||||
fake_count=2)}
|
||||
deltas = {RESOURCE: 1}
|
||||
self.plugin.update_quota_limit(self.context, PROJECT, RESOURCE, 2)
|
||||
self.assertRaises(exceptions.OverQuota,
|
||||
quota_driver.make_reservation,
|
||||
self.context,
|
||||
self.context.tenant_id,
|
||||
resources,
|
||||
deltas,
|
||||
self.plugin)
|
||||
|
||||
@ -344,6 +344,24 @@ class QuotaExtensionDbTestCase(QuotaExtensionTestCase):
|
||||
extra_environ=env, expect_errors=True)
|
||||
self.assertEqual(400, res.status_int)
|
||||
|
||||
def test_make_reservation_resource_unknown_raises(self):
|
||||
tenant_id = 'tenant_id1'
|
||||
self.assertRaises(exceptions.QuotaResourceUnknown,
|
||||
quota.QUOTAS.make_reservation,
|
||||
context.get_admin_context(load_admin_roles=False),
|
||||
tenant_id,
|
||||
{'foobar': 1},
|
||||
plugin=None)
|
||||
|
||||
def test_make_reservation_negative_delta_raises(self):
|
||||
tenant_id = 'tenant_id1'
|
||||
self.assertRaises(exceptions.InvalidQuotaValue,
|
||||
quota.QUOTAS.make_reservation,
|
||||
context.get_admin_context(load_admin_roles=False),
|
||||
tenant_id,
|
||||
{'network': -1},
|
||||
plugin=None)
|
||||
|
||||
|
||||
class QuotaExtensionCfgTestCase(QuotaExtensionTestCase):
|
||||
fmt = 'json'
|
||||
|
||||
@ -165,7 +165,8 @@ class TestTrackedResource(testlib_api.SqlTestCaseLight):
|
||||
res.count(self.context, None, self.tenant_id,
|
||||
resync_usage=True)
|
||||
mock_set_quota_usage.assert_called_once_with(
|
||||
self.context, self.resource, self.tenant_id, in_use=2)
|
||||
self.context, self.resource, self.tenant_id,
|
||||
reserved=0, in_use=2)
|
||||
|
||||
def test_count_with_dirty_true_no_usage_info(self):
|
||||
res = self._create_resource()
|
||||
@ -184,7 +185,8 @@ class TestTrackedResource(testlib_api.SqlTestCaseLight):
|
||||
self.tenant_id)
|
||||
res.count(self.context, None, self.tenant_id, resync_usage=True)
|
||||
mock_set_quota_usage.assert_called_once_with(
|
||||
self.context, self.resource, self.tenant_id, in_use=2)
|
||||
self.context, self.resource, self.tenant_id,
|
||||
reserved=0, in_use=2)
|
||||
|
||||
def test_add_delete_data_triggers_event(self):
|
||||
res = self._create_resource()
|
||||
@ -251,4 +253,5 @@ class TestTrackedResource(testlib_api.SqlTestCaseLight):
|
||||
# and now it should be in sync
|
||||
self.assertNotIn(self.tenant_id, res._out_of_sync_tenants)
|
||||
mock_set_quota_usage.assert_called_once_with(
|
||||
self.context, self.resource, self.tenant_id, in_use=2)
|
||||
self.context, self.resource, self.tenant_id,
|
||||
reserved=0, in_use=2)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user