Quota management for Nova-APIGW and Cinder-APIGW(part1)

Port the models and db access code mostly from Cinder, for
Cinder has implemented for hierarchy multi-tenancy quota management

The quota management and control in Tricircle is described in the
design doc:
https://docs.google.com/document/d/18kZZ1snMOCD9IQvUKI5NVDzSASpw-QKj7l2zNqMEd3g/

BP: https://blueprints.launchpad.net/tricircle/+spec/implement-stateless

Change-Id: I90d210f2154d8f68be49cc4fc4a62f2532cb5f92
Signed-off-by: Chaoyi Huang <joehuang@huawei.com>
This commit is contained in:
Chaoyi Huang 2016-01-21 17:18:12 +08:00
parent 81b45f2c1d
commit 1da8d2a473
9 changed files with 1394 additions and 26 deletions

View File

@ -13,6 +13,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
from pecan import request
import oslo_context.context as oslo_ctx
@ -119,3 +121,13 @@ class Context(ContextBase):
if not self._session:
self._session = core.get_session()
return self._session
def elevated(self, read_deleted=None, overwrite=False):
"""Return a version of this context with admin flag set."""
ctx = copy.copy(self)
ctx.is_admin = True
if read_deleted is not None:
ctx.read_deleted = read_deleted
return ctx

View File

@ -56,7 +56,9 @@ class BadRequest(TricircleException):
class NotFound(TricircleException):
pass
message = _("Resource could not be found.")
code = 404
safe = True
class Conflict(TricircleException):
@ -120,3 +122,45 @@ class ResourceNotSupported(TricircleException):
def __init__(self, resource, method):
super(ResourceNotSupported, self).__init__(resource=resource,
method=method)
class Invalid(TricircleException):
message = _("Unacceptable parameters.")
code = 400
class InvalidReservationExpiration(Invalid):
message = _("Invalid reservation expiration %(expire)s.")
class InvalidQuotaValue(Invalid):
message = _("Change would make usage less than 0 for the following "
"resources: %(unders)s")
class QuotaNotFound(NotFound):
message = _("Quota could not be found")
class QuotaResourceUnknown(QuotaNotFound):
message = _("Unknown quota resources %(unknown)s.")
class ProjectQuotaNotFound(QuotaNotFound):
message = _("Quota for project %(project_id)s could not be found.")
class QuotaClassNotFound(QuotaNotFound):
message = _("Quota class %(class_name)s could not be found.")
class QuotaUsageNotFound(QuotaNotFound):
message = _("Quota usage for project %(project_id)s could not be found.")
class ReservationNotFound(QuotaNotFound):
message = _("Quota reservation %(uuid)s could not be found.")
class OverQuota(TricircleException):
message = _("Quota exceeded for resources: %(overs)s")

194
tricircle/common/quota.py Normal file
View File

@ -0,0 +1,194 @@
# Copyright 2015 Huawei Technologies Co., Ltd.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Routines for configuring tricircle, largely copy from Neutron
"""
from oslo_config import cfg
import oslo_log.log as logging
from tricircle.common import exceptions
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class BaseResource(object):
"""Describe a single resource for quota checking."""
def __init__(self, name, flag=None, parent_project_id=None):
"""Initializes a Resource.
:param name: The name of the resource, i.e., "volumes".
:param flag: The name of the flag or configuration option
which specifies the default value of the quota
for this resource.
:param parent_project_id: The id of the current project's parent,
if any.
"""
self.name = name
self.flag = flag
self.parent_project_id = parent_project_id
def quota(self, driver, context, **kwargs):
"""Given a driver and context, obtain the quota for this resource.
:param driver: A quota driver.
:param context: The request context.
:param project_id: The project to obtain the quota value for.
If not provided, it is taken from the
context. If it is given as None, no
project-specific quota will be searched
for.
:param quota_class: The quota class corresponding to the
project, or for which the quota is to be
looked up. If not provided, it is taken
from the context. If it is given as None,
no quota class-specific quota will be
searched for. Note that the quota class
defaults to the value in the context,
which may not correspond to the project if
project_id is not the same as the one in
the context.
"""
# Get the project ID
project_id = kwargs.get('project_id', context.project_id)
# Ditto for the quota class
quota_class = kwargs.get('quota_class', context.quota_class)
# Look up the quota for the project
if project_id:
try:
return driver.get_by_project(context, project_id, self.name)
except exceptions.ProjectQuotaNotFound:
pass
# Try for the quota class
if quota_class:
try:
return driver.get_by_class(context, quota_class, self.name)
except exceptions.QuotaClassNotFound:
pass
# OK, return the default
return driver.get_default(context, self,
parent_project_id=self.parent_project_id)
@property
def default(self):
"""Return the default value of the quota."""
if self.parent_project_id:
return 0
return CONF[self.flag] if self.flag else -1
class ReservableResource(BaseResource):
"""Describe a reservable resource."""
def __init__(self, name, sync, flag=None):
"""Initializes a ReservableResource.
Reservable resources are those resources which directly
correspond to objects in the database, i.e., volumes, gigabytes,
etc. A ReservableResource must be constructed with a usage
synchronization function, which will be called to determine the
current counts of one or more resources.
The usage synchronization function will be passed three
arguments: an admin context, the project ID, and an opaque
session object, which should in turn be passed to the
underlying database function. Synchronization functions
should return a dictionary mapping resource names to the
current in_use count for those resources; more than one
resource and resource count may be returned. Note that
synchronization functions may be associated with more than one
ReservableResource.
:param name: The name of the resource, i.e., "volumes".
:param sync: A dbapi methods name which returns a dictionary
to resynchronize the in_use count for one or more
resources, as described above.
:param flag: The name of the flag or configuration option
which specifies the default value of the quota
for this resource.
"""
super(ReservableResource, self).__init__(name, flag=flag)
if sync:
self.sync = sync
class AbsoluteResource(BaseResource):
"""Describe a non-reservable resource."""
pass
class CountableResource(AbsoluteResource):
"""Describe a resource where counts aren't based only on the project ID."""
def __init__(self, name, count, flag=None):
"""Initializes a CountableResource.
Countable resources are those resources which directly
correspond to objects in the database, i.e., volumes, gigabytes,
etc., but for which a count by project ID is inappropriate. A
CountableResource must be constructed with a counting
function, which will be called to determine the current counts
of the resource.
The counting function will be passed the context, along with
the extra positional and keyword arguments that are passed to
Quota.count(). It should return an integer specifying the
count.
Note that this counting is not performed in a transaction-safe
manner. This resource class is a temporary measure to provide
required functionality, until a better approach to solving
this problem can be evolved.
:param name: The name of the resource, i.e., "volumes".
:param count: A callable which returns the count of the
resource. The arguments passed are as described
above.
:param flag: The name of the flag or configuration option
which specifies the default value of the quota
for this resource.
"""
super(CountableResource, self).__init__(name, flag=flag)
self.count = count
class VolumeTypeResource(ReservableResource):
"""ReservableResource for a specific volume type."""
def __init__(self, part_name, volume_type):
"""Initializes a VolumeTypeResource.
:param part_name: The kind of resource, i.e., "volumes".
:param volume_type: The volume type for this resource.
"""
self.volume_type_name = volume_type['name']
self.volume_type_id = volume_type['id']
name = "%s_%s" % (part_name, self.volume_type_name)
super(VolumeTypeResource, self).__init__(name, "_sync_%s" % part_name)

View File

@ -13,11 +13,28 @@
# License for the specific language governing permissions and limitations
# under the License.
import functools
import time
import uuid
from oslo_config import cfg
from oslo_db import exception as db_exc
from oslo_log import log as logging
from oslo_utils import timeutils
from tricircle.common.context import is_admin_context as _is_admin_context
from tricircle.common import exceptions
from tricircle.common.i18n import _
from tricircle.common.i18n import _LW
from tricircle.db import core
from tricircle.db import models
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
def create_pod(context, pod_dict):
with context.session.begin():
return core.create_resource(context, models.Pod, pod_dict)
@ -166,3 +183,606 @@ def get_pod_by_name(context, pod_name):
return pod
return None
_DEFAULT_QUOTA_NAME = 'default'
def _is_user_context(context):
"""Indicates if the request context is a normal user."""
if not context:
return False
if context.is_admin:
return False
if not context.user_id or not context.project_id:
return False
return True
def authorize_quota_class_context(context, class_name):
"""Ensures a request has permission to access the given quota class."""
if _is_user_context(context):
if not context.quota_class:
raise exceptions.NotAuthorized()
elif context.quota_class != class_name:
raise exceptions.NotAuthorized()
def authorize_project_context(context, project_id):
"""Ensures a request has permission to access the given project."""
if _is_user_context(context):
if not context.project_id:
raise exceptions.NotAuthorized()
elif context.project_id != project_id:
raise exceptions.NotAuthorized()
def authorize_user_context(context, user_id):
"""Ensures a request has permission to access the given user."""
if _is_user_context(context):
if not context.user_id:
raise exceptions.NotAuthorized()
elif context.user_id != user_id:
raise exceptions.NotAuthorized()
def require_admin_context(f):
"""Decorator to require admin request context.
The first argument to the wrapped function must be the context.
"""
def wrapper(*args, **kwargs):
if not _is_admin_context(args[0]):
raise exceptions.AdminRequired()
return f(*args, **kwargs)
return wrapper
def require_context(f):
"""Decorator to require *any* user or admin context.
This does no authorization for user or project access matching, see
:py:func:`authorize_project_context` and
:py:func:`authorize_user_context`.
The first argument to the wrapped function must be the context.
"""
def wrapper(*args, **kwargs):
if not _is_admin_context(args[0]) and not _is_user_context(args[0]):
raise exceptions.NotAuthorized()
return f(*args, **kwargs)
return wrapper
def _retry_on_deadlock(f):
"""Decorator to retry a DB API call if Deadlock was received."""
@functools.wraps(f)
def wrapped(*args, **kwargs):
while True:
try:
return f(*args, **kwargs)
except db_exc.DBDeadlock:
LOG.warning(_LW("Deadlock detected when running "
"'%(func_name)s': Retrying..."),
dict(func_name=f.__name__))
# Retry!
time.sleep(0.5)
continue
functools.update_wrapper(wrapped, f)
return wrapped
def model_query(context, *args, **kwargs):
"""Query helper that accounts for context's `read_deleted` field.
:param context: context to query under
:param session: if present, the session to use
:param read_deleted: if present, overrides context's read_deleted field.
:param project_only: if present and context is user-type, then restrict
query to match the context's project_id.
"""
session = kwargs.get('session') or context.session
read_deleted = kwargs.get('read_deleted') or context.read_deleted
project_only = kwargs.get('project_only')
query = session.query(*args)
if read_deleted == 'no':
query = query.filter_by(deleted=False)
elif read_deleted == 'yes':
pass # omit the filter to include deleted and active
elif read_deleted == 'only':
query = query.filter_by(deleted=True)
elif read_deleted == 'int_no':
query = query.filter_by(deleted=0)
else:
raise Exception(
_("Unrecognized read_deleted value '%s'") % read_deleted)
if project_only and _is_user_context(context):
query = query.filter_by(project_id=context.project_id)
return query
@require_context
def _quota_get(context, project_id, resource, session=None):
result = model_query(context, models.Quotas, session=session,
read_deleted="no").\
filter_by(project_id=project_id).\
filter_by(resource=resource).\
first()
if not result:
raise exceptions.ProjectQuotaNotFound(project_id=project_id)
return result
@require_context
def quota_get(context, project_id, resource):
return _quota_get(context, project_id, resource)
@require_context
def quota_get_all_by_project(context, project_id):
authorize_project_context(context, project_id)
rows = model_query(context, models.Quotas, read_deleted="no").\
filter_by(project_id=project_id).\
all()
result = {'project_id': project_id}
for row in rows:
result[row.resource] = row.hard_limit
return result
@require_context
def quota_allocated_get_all_by_project(context, project_id):
rows = model_query(context, models.Quotas, read_deleted='no').filter_by(
project_id=project_id).all()
result = {'project_id': project_id}
for row in rows:
result[row.resource] = row.allocated
return result
@require_admin_context
def quota_create(context, project_id, resource, limit, allocated=0):
quota_ref = models.Quotas()
quota_ref.project_id = project_id
quota_ref.resource = resource
quota_ref.hard_limit = limit
if allocated:
quota_ref.allocated = allocated
session = core.get_session()
with session.begin():
quota_ref.save(session)
return quota_ref
@require_admin_context
def quota_update(context, project_id, resource, limit):
with context.session.begin():
quota_ref = _quota_get(context, project_id, resource,
session=context.session)
quota_ref.hard_limit = limit
return quota_ref
@require_admin_context
def quota_allocated_update(context, project_id, resource, allocated):
with context.session.begin():
quota_ref = _quota_get(context, project_id, resource,
session=context.session)
quota_ref.allocated = allocated
return quota_ref
@require_admin_context
def quota_destroy(context, project_id, resource):
with context.session.begin():
quota_ref = _quota_get(context, project_id, resource,
session=context.session)
quota_ref.delete(session=context.session)
@require_context
def _quota_class_get(context, class_name, resource, session=None):
result = model_query(context, models.QuotaClasses, session=session,
read_deleted="no").\
filter_by(class_name=class_name).\
filter_by(resource=resource).\
first()
if not result:
raise exceptions.QuotaClassNotFound(class_name=class_name)
return result
@require_context
def quota_class_get(context, class_name, resource):
return _quota_class_get(context, class_name, resource)
def quota_class_get_default(context):
rows = model_query(context, models.QuotaClasses,
read_deleted="no").\
filter_by(class_name=_DEFAULT_QUOTA_NAME).all()
result = {'class_name': _DEFAULT_QUOTA_NAME}
for row in rows:
result[row.resource] = row.hard_limit
return result
@require_context
def quota_class_get_all_by_name(context, class_name):
authorize_quota_class_context(context, class_name)
rows = model_query(context, models.QuotaClasses, read_deleted="no").\
filter_by(class_name=class_name).\
all()
result = {'class_name': class_name}
for row in rows:
result[row.resource] = row.hard_limit
return result
@require_admin_context
def quota_class_create(context, class_name, resource, limit):
quota_class_ref = models.QuotaClasses()
quota_class_ref.class_name = class_name
quota_class_ref.resource = resource
quota_class_ref.hard_limit = limit
session = core.get_session()
with session.begin():
quota_class_ref.save(session)
return quota_class_ref
@require_admin_context
def quota_class_update(context, class_name, resource, limit):
with context.session.begin():
quota_class_ref = _quota_class_get(context, class_name, resource,
session=context.session)
quota_class_ref.hard_limit = limit
return quota_class_ref
@require_admin_context
def quota_class_destroy(context, class_name, resource):
with context.session.begin():
quota_class_ref = _quota_class_get(context, class_name, resource,
session=context.session)
quota_class_ref.delete(session=context.session)
@require_admin_context
def quota_class_destroy_all_by_name(context, class_name):
with context.session.begin():
quota_classes = model_query(context, models.QuotaClasses,
session=context.session,
read_deleted="no").\
filter_by(class_name=class_name).\
all()
for quota_class_ref in quota_classes:
quota_class_ref.delete(session=context.session)
@require_context
def quota_usage_get(context, project_id, resource):
result = model_query(context, models.QuotaUsages, read_deleted="no").\
filter_by(project_id=project_id).\
filter_by(resource=resource).\
first()
if not result:
raise exceptions.QuotaUsageNotFound(project_id=project_id)
return result
@require_context
def quota_usage_get_all_by_project(context, project_id):
authorize_project_context(context, project_id)
rows = model_query(context, models.QuotaUsages, read_deleted="no").\
filter_by(project_id=project_id).\
all()
result = {'project_id': project_id}
for row in rows:
result[row.resource] = dict(in_use=row.in_use, reserved=row.reserved)
return result
@require_admin_context
def _quota_usage_create(context, project_id, resource, in_use, reserved,
until_refresh, session=None):
quota_usage_ref = models.QuotaUsages()
quota_usage_ref.project_id = project_id
quota_usage_ref.resource = resource
quota_usage_ref.in_use = in_use
quota_usage_ref.reserved = reserved
quota_usage_ref.until_refresh = until_refresh
quota_usage_ref.save(session=session)
return quota_usage_ref
def _reservation_create(context, uuid, usage, project_id, resource, delta,
expire, session=None):
reservation_ref = models.Reservation()
reservation_ref.uuid = uuid
reservation_ref.usage_id = usage['id']
reservation_ref.project_id = project_id
reservation_ref.resource = resource
reservation_ref.delta = delta
reservation_ref.expire = expire
reservation_ref.save(session=session)
return reservation_ref
# NOTE(johannes): The quota code uses SQL locking to ensure races don't
# cause under or over counting of resources. To avoid deadlocks, this
# code always acquires the lock on quota_usages before acquiring the lock
# on reservations.
def _get_quota_usages(context, session, project_id):
# Broken out for testability
rows = model_query(context, models.QuotaUsages,
read_deleted="no",
session=session).\
filter_by(project_id=project_id).\
with_lockmode('update').\
all()
return {row.resource: row for row in rows}
def _get_quota_usages_by_resource(context, session, project_id, resource):
# TODO(joehuang), add user_id as part of the filter
rows = model_query(context, models.QuotaUsages,
read_deleted="no",
session=session).\
filter_by(project_id=project_id).\
filter_by(resource=resource).\
with_lockmode('update').\
all()
return {row.resource: row for row in rows}
@require_context
@_retry_on_deadlock
def quota_reserve(context, resources, quotas, deltas, expire,
until_refresh, max_age, project_id=None):
elevated = context.elevated()
with context.session.begin():
if project_id is None:
project_id = context.project_id
# Get the current usages
usages = _get_quota_usages(context, context.session, project_id)
# Handle usage refresh
refresh = False
work = set(deltas.keys())
while work:
resource = work.pop()
# Do we need to refresh the usage?
if resource not in usages:
usages[resource] = _quota_usage_create(elevated,
project_id,
resource,
0, 0,
until_refresh or None,
session=context.session)
refresh = True
elif usages[resource].in_use < 0:
# Negative in_use count indicates a desync, so try to
# heal from that...
refresh = True
elif usages[resource].until_refresh is not None:
usages[resource].until_refresh -= 1
if usages[resource].until_refresh <= 0:
refresh = True
elif max_age and usages[resource].updated_at is not None and (
(usages[resource].updated_at -
timeutils.utcnow()).seconds >= max_age):
refresh = True
if refresh:
# refresh from the bottom pod
pass
# Check for deltas that would go negative
unders = [r for r, delta in deltas.items()
if delta < 0 and delta + usages[r].in_use < 0]
# Now, let's check the quotas
# NOTE(Vek): We're only concerned about positive increments.
# If a project has gone over quota, we want them to
# be able to reduce their usage without any
# problems.
overs = [r for r, delta in deltas.items()
if quotas[r] >= 0 and delta >= 0 and
quotas[r] < delta + usages[r].in_use + usages[r].reserved]
# NOTE(Vek): The quota check needs to be in the transaction,
# but the transaction doesn't fail just because
# we're over quota, so the OverQuota raise is
# outside the transaction. If we did the raise
# here, our usage updates would be discarded, but
# they're not invalidated by being over-quota.
# Create the reservations
if not overs:
reservations = []
for resource, delta in deltas.items():
reservation = _reservation_create(elevated,
str(uuid.uuid4()),
usages[resource],
project_id,
resource, delta, expire,
session=context.session)
reservations.append(reservation.uuid)
# Also update the reserved quantity
# NOTE(Vek): Again, we are only concerned here about
# positive increments. Here, though, we're
# worried about the following scenario:
#
# 1) User initiates resize down.
# 2) User allocates a new instance.
# 3) Resize down fails or is reverted.
# 4) User is now over quota.
#
# To prevent this, we only update the
# reserved value if the delta is positive.
if delta > 0:
usages[resource].reserved += delta
if unders:
LOG.warning(_LW("Change will make usage less than 0 for the following "
"resources: %s"), unders)
if overs:
usages = {k: dict(in_use=v['in_use'], reserved=v['reserved'])
for k, v in usages.items()}
raise exceptions.OverQuota(overs=sorted(overs), quotas=quotas,
usages=usages)
return reservations
def _quota_reservations(session, context, reservations):
"""Return the relevant reservations."""
# Get the listed reservations
return model_query(context, models.Reservation,
read_deleted="no",
session=session).\
filter(models.Reservation.uuid.in_(reservations)).\
with_lockmode('update').\
all()
@require_context
@_retry_on_deadlock
def reservation_commit(context, reservations, project_id=None):
with context.session.begin():
usages = _get_quota_usages(context, context.session, project_id)
for reservation in _quota_reservations(context.session,
context,
reservations):
usage = usages[reservation.resource]
if reservation.delta >= 0:
usage.reserved -= reservation.delta
usage.in_use += reservation.delta
reservation.delete(session=context.session)
@require_context
@_retry_on_deadlock
def reservation_rollback(context, reservations, project_id=None):
with context.session.begin():
usages = _get_quota_usages(context, context.session, project_id)
for reservation in _quota_reservations(context.session,
context,
reservations):
usage = usages[reservation.resource]
if reservation.delta >= 0:
usage.reserved -= reservation.delta
reservation.delete(session=context.session)
def quota_destroy_by_project(*args, **kwargs):
"""Destroy all limit quotas associated with a project.
Leaves usage and reservation quotas intact.
"""
quota_destroy_all_by_project(only_quotas=True, *args, **kwargs)
@require_admin_context
@_retry_on_deadlock
def quota_destroy_all_by_project(context, project_id, only_quotas=False):
"""Destroy all quotas associated with a project.
This includes limit quotas, usage quotas and reservation quotas.
Optionally can only remove limit quotas and leave other types as they are.
:param context: The request context, for access checks.
:param project_id: The ID of the project being deleted.
:param only_quotas: Only delete limit quotas, leave other types intact.
"""
with context.session.begin():
quotas = model_query(context, models.Quotas, session=context.session,
read_deleted="no").\
filter_by(project_id=project_id).\
all()
for quota_ref in quotas:
quota_ref.delete(session=context.session)
if only_quotas:
return
quota_usages = model_query(context, models.QuotaUsages,
session=context.session,
read_deleted="no").\
filter_by(project_id=project_id).\
all()
for quota_usage_ref in quota_usages:
quota_usage_ref.delete(session=context.session)
reservations = model_query(context, models.Reservation,
session=context.session,
read_deleted="no").\
filter_by(project_id=project_id).\
all()
for reservation_ref in reservations:
reservation_ref.delete(session=context.session)
@require_admin_context
@_retry_on_deadlock
def reservation_expire(context):
with context.session.begin():
current_time = timeutils.utcnow()
results = model_query(context, models.Reservation,
session=context.session,
read_deleted="no").\
filter(models.Reservation.expire < current_time).\
all()
if results:
for reservation in results:
if reservation.delta >= 0:
reservation.usage.reserved -= reservation.delta
reservation.usage.save(session=context.session)
reservation.delete(session=context.session)

View File

@ -14,13 +14,17 @@
# under the License.
import threading
import sqlalchemy as sql
from sqlalchemy.ext import declarative
from sqlalchemy.inspection import inspect
from oslo_config import cfg
import oslo_db.options as db_options
import oslo_db.sqlalchemy.session as db_session
from oslo_utils import strutils
import sqlalchemy as sql
from sqlalchemy.ext import declarative
from sqlalchemy.inspection import inspect
from tricircle.common import exceptions
@ -31,6 +35,7 @@ db_opts = [
]
cfg.CONF.register_opts(db_opts)
_LOCK = threading.Lock()
_engine_facade = None
ModelBase = declarative.declarative_base()
@ -64,11 +69,14 @@ def _filter_query(model, query, filters):
def _get_engine_facade():
global _LOCK
with _LOCK:
global _engine_facade
if not _engine_facade:
t_connection = cfg.CONF.tricircle_db_connection
_engine_facade = db_session.EngineFacade(t_connection, _conf=cfg.CONF)
_engine_facade = db_session.EngineFacade(t_connection,
_conf=cfg.CONF)
return _engine_facade

View File

@ -118,14 +118,72 @@ def upgrade(migrate_engine):
quotas = sql.Table(
'quotas', meta,
sql.Column('id', sql.Integer, primary_key=True),
sql.Column('project_id', sql.String(255)),
sql.Column('project_id', sql.String(255), index=True),
sql.Column('resource', sql.String(255), nullable=False),
sql.Column('hard_limit', sql.Integer),
sql.Column('allocated', sql.Integer, default=0),
sql.Column('created_at', sql.DateTime),
sql.Column('updated_at', sql.DateTime),
sql.Column('deleted_at', sql.DateTime),
sql.Column('deleted', sql.Integer),
migrate.UniqueConstraint(
'project_id', 'resource', 'deleted',
name='uniq_quotas0project_id0resource0deleted'),
mysql_engine='InnoDB',
mysql_charset='utf8')
quota_classes = sql.Table(
'quota_classes', meta,
sql.Column('id', sql.Integer, primary_key=True),
sql.Column('class_name', sql.String(255), index=True),
sql.Column('resource', sql.String(255), nullable=False),
sql.Column('hard_limit', sql.Integer),
sql.Column('created_at', sql.DateTime),
sql.Column('updated_at', sql.DateTime),
sql.Column('deleted_at', sql.DateTime),
sql.Column('deleted', sql.Integer),
migrate.UniqueConstraint(
'project_id', 'resource',
name='uniq_quotas0project_id0resource'),
'class_name', 'resource', 'deleted',
name='uniq_quota_classes0class_name0resource0deleted'),
mysql_engine='InnoDB',
mysql_charset='utf8')
quota_usages = sql.Table(
'quota_usages', meta,
sql.Column('id', sql.Integer, primary_key=True),
sql.Column('project_id', sql.String(255), index=True),
sql.Column('user_id', sql.String(255), index=True),
sql.Column('resource', sql.String(255), nullable=False),
sql.Column('in_use', sql.Integer),
sql.Column('reserved', sql.Integer),
sql.Column('until_refresh', sql.Integer),
sql.Column('created_at', sql.DateTime),
sql.Column('updated_at', sql.DateTime),
sql.Column('deleted_at', sql.DateTime),
sql.Column('deleted', sql.Integer),
mysql_engine='InnoDB',
mysql_charset='utf8')
reservations = sql.Table(
'reservations', meta,
sql.Column('id', sql.Integer(), primary_key=True),
sql.Column('uuid', sql.String(length=36), nullable=False),
sql.Column('usage_id', sql.Integer(),
sql.ForeignKey('quota_usages.id'),
nullable=False),
sql.Column('project_id',
sql.String(length=255),
index=True),
sql.Column('resource',
sql.String(length=255)),
sql.Column('delta', sql.Integer(), nullable=False),
sql.Column('expire', sql.DateTime),
sql.Column('created_at', sql.DateTime),
sql.Column('updated_at', sql.DateTime),
sql.Column('deleted_at', sql.DateTime),
sql.Column('deleted', sql.Boolean(create_constraint=True,
name=None)),
mysql_engine='InnoDB',
mysql_charset='utf8')
@ -167,8 +225,9 @@ def upgrade(migrate_engine):
tables = [aggregates, aggregate_metadata, instance_types,
instance_type_projects, instance_type_extra_specs, key_pairs,
quotas, volume_types, quality_of_service_specs,
cascaded_pods_resource_routing]
quotas, quota_classes, quota_usages, reservations,
volume_types,
quality_of_service_specs, cascaded_pods_resource_routing]
for table in tables:
table.create()
@ -178,6 +237,8 @@ def upgrade(migrate_engine):
'references': [instance_types.c.id]},
{'columns': [instance_type_extra_specs.c.instance_type_id],
'references': [instance_types.c.id]},
{'columns': [reservations.c.usage_id],
'references': [quota_usages.c.id]},
{'columns': [volume_types.c.qos_specs_id],
'references': [quality_of_service_specs.c.id]},
{'columns': [quality_of_service_specs.c.specs_id],

View File

@ -13,12 +13,14 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo_db.sqlalchemy import models
import sqlalchemy as sql
from sqlalchemy.dialects import mysql
from sqlalchemy.orm import relationship
from sqlalchemy import schema
from oslo_db.sqlalchemy import models
from oslo_utils import timeutils
from tricircle.db import core
@ -143,7 +145,26 @@ class KeyPair(core.ModelBase, core.DictBase, models.TimestampMixin):
nullable=False, server_default='ssh')
class Quota(core.ModelBase, core.DictBase, models.TimestampMixin):
# Quota part are ported from Cinder for hierarchy multi-tenancy quota control
class QuotasBase(models.ModelBase, core.DictBase,
models.TimestampMixin, models.SoftDeleteMixin):
"""QuotasBase.
provide base class for quota series tables. For it inherits from
models.ModelBase, this is different from other tables
"""
__table_args__ = {'mysql_engine': 'InnoDB'}
metadata = None
def delete(self, session):
"""Delete this object."""
self.deleted = True
self.deleted_at = timeutils.utcnow()
self.save(session=session)
class Quotas(core.ModelBase, QuotasBase):
"""Represents a single quota override for a project.
If there is no row for a given project id and resource, then the
@ -154,16 +175,96 @@ class Quota(core.ModelBase, core.DictBase, models.TimestampMixin):
"""
__tablename__ = 'quotas'
__table_args__ = (
schema.UniqueConstraint('project_id', 'resource',
name='uniq_quotas0project_id0resource'),
)
attributes = ['id', 'project_id', 'resource', 'hard_limit',
'created_at', 'updated_at']
schema.UniqueConstraint(
'project_id', 'resource', 'deleted',
name='uniq_quotas0project_id0resource0deleted'),)
attributes = ['id', 'project_id', 'resource',
'hard_limit', 'allocated',
'created_at', 'updated_at', 'deleted_at', 'deleted']
id = sql.Column(sql.Integer, primary_key=True)
project_id = sql.Column(sql.String(255))
project_id = sql.Column(sql.String(255), index=True)
resource = sql.Column(sql.String(255), nullable=False)
hard_limit = sql.Column(sql.Integer)
allocated = sql.Column(sql.Integer, default=0)
class QuotaClasses(core.ModelBase, QuotasBase):
"""Quota_classes.
make default quota as a update-able quota.Mainly for the command
quota-class-show and quota-class-update
"""
__tablename__ = 'quota_classes'
__table_args__ = (
schema.UniqueConstraint(
'class_name', 'resource', 'deleted',
name='uniq_quota_classes0class_name0resource0deleted'),
)
attributes = ['id', 'class_name', 'resource', 'hard_limit',
'created_at', 'updated_at', 'deleted_at', 'deleted']
id = sql.Column(sql.Integer, primary_key=True)
class_name = sql.Column(sql.String(255), index=True)
resource = sql.Column(sql.String(255), nullable=False)
hard_limit = sql.Column(sql.Integer)
class QuotaUsages(core.ModelBase, QuotasBase):
"""Quota_uages.
store quota usages for project resource
"""
__tablename__ = 'quota_usages'
__table_args__ = ()
attributes = ['id', 'project_id', 'user_id', 'resource',
'in_use', 'reserved', 'until_refresh',
'created_at', 'updated_at', 'deleted_at', 'deleted']
id = sql.Column(sql.Integer, primary_key=True)
project_id = sql.Column(sql.String(255), index=True)
user_id = sql.Column(sql.String(255), index=True)
resource = sql.Column(sql.String(255), nullable=False)
in_use = sql.Column(sql.Integer)
reserved = sql.Column(sql.Integer, default=0)
until_refresh = sql.Column(sql.Integer, default=0)
@property
def total(self):
return self.in_use + self.reserved
class Reservation(core.ModelBase, QuotasBase):
"""Reservation.
Represents a resource reservation for quotas
"""
__tablename__ = 'reservations'
__table_args__ = ()
attributes = ['id', 'uuid', 'usage_id', 'project_id', 'resource',
'delta', 'expire',
'created_at', 'updated_at', 'deleted_at', 'deleted']
id = sql.Column(sql.Integer, primary_key=True)
uuid = sql.Column(sql.String(36), nullable=False)
usage_id = sql.Column(sql.Integer,
sql.ForeignKey('quota_usages.id'),
nullable=False)
project_id = sql.Column(sql.String(255), index=True)
resource = sql.Column(sql.String(255))
delta = sql.Column(sql.Integer)
expire = sql.Column(sql.DateTime, nullable=False)
usage = relationship(
"QuotaUsages",
foreign_keys=usage_id,
primaryjoin='and_(Reservation.usage_id == QuotaUsages.id,'
'QuotaUsages.deleted == 0)')
class VolumeTypes(core.ModelBase, core.DictBase, models.TimestampMixin):

View File

@ -13,10 +13,14 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import six
import unittest
from tricircle.common import context
from tricircle.common import exceptions
from tricircle.common import quota
from tricircle.db import api
from tricircle.db import core
from tricircle.db import models
@ -181,3 +185,323 @@ class APITest(unittest.TestCase):
def tearDown(self):
core.ModelBase.metadata.drop_all(core.get_engine())
class QuotaApiTestCase(unittest.TestCase):
def setUp(self):
core.initialize()
core.ModelBase.metadata.create_all(core.get_engine())
self.context = context.get_admin_context()
def _quota_reserve(self, context, project_id):
"""Create sample Quota, QuotaUsage and Reservation objects.
There is no method api.quota_usage_create(), so we have to use
api.quota_reserve() for creating QuotaUsage objects.
Returns reservations uuids.
"""
quotas = {}
resources = {}
deltas = {}
for i, resource in enumerate(('volumes', 'gigabytes')):
quota_obj = api.quota_create(context, project_id, resource, i + 1)
quotas[resource] = quota_obj.hard_limit
resources[resource] = quota.ReservableResource(resource, None)
deltas[resource] = i + 1
return api.quota_reserve(
context, resources, quotas, deltas,
datetime.datetime.utcnow(), datetime.datetime.utcnow(),
datetime.timedelta(days=1), project_id
)
def _dict_from_object(self, obj, ignored_keys):
if ignored_keys is None:
ignored_keys = []
if isinstance(obj, dict):
items = obj.items()
else:
items = obj.iteritems()
return {k: v for k, v in items
if k not in ignored_keys}
def _assertEqualObjects(self, obj1, obj2, ignored_keys=None):
obj1 = self._dict_from_object(obj1, ignored_keys)
obj2 = self._dict_from_object(obj2, ignored_keys)
self.assertEqual(
len(obj1), len(obj2),
"Keys mismatch: %s" % six.text_type(
set(obj1.keys()) ^ set(obj2.keys())))
for key, value in obj1.items():
self.assertEqual(value, obj2[key])
def tearDown(self):
core.ModelBase.metadata.drop_all(core.get_engine())
class DBAPIReservationTestCase(QuotaApiTestCase):
"""Tests for db.api.reservation_* methods."""
def setUp(self):
super(DBAPIReservationTestCase, self).setUp()
self.values = {
'uuid': 'sample-uuid',
'project_id': 'project1',
'resource': 'resource',
'delta': 42,
'expire': (datetime.datetime.utcnow() +
datetime.timedelta(days=1)),
'usage': {'id': 1}
}
def test_reservation_commit(self):
reservations = self._quota_reserve(self.context, 'project1')
expected = {'project_id': 'project1',
'volumes': {'reserved': 1, 'in_use': 0},
'gigabytes': {'reserved': 2, 'in_use': 0},
}
self.assertEqual(expected,
api.quota_usage_get_all_by_project(
self.context, 'project1'))
api.reservation_commit(self.context, reservations, 'project1')
expected = {'project_id': 'project1',
'volumes': {'reserved': 0, 'in_use': 1},
'gigabytes': {'reserved': 0, 'in_use': 2},
}
self.assertEqual(expected,
api.quota_usage_get_all_by_project(
self.context,
'project1'))
def test_reservation_rollback(self):
reservations = self._quota_reserve(self.context, 'project1')
expected = {'project_id': 'project1',
'volumes': {'reserved': 1, 'in_use': 0},
'gigabytes': {'reserved': 2, 'in_use': 0},
}
self.assertEqual(expected,
api.quota_usage_get_all_by_project(
self.context,
'project1'))
api.reservation_rollback(self.context, reservations, 'project1')
expected = {'project_id': 'project1',
'volumes': {'reserved': 0, 'in_use': 0},
'gigabytes': {'reserved': 0, 'in_use': 0},
}
self.assertEqual(expected,
api.quota_usage_get_all_by_project(
self.context,
'project1'))
def test_reservation_expire(self):
self.values['expire'] = datetime.datetime.utcnow() + \
datetime.timedelta(days=1)
self._quota_reserve(self.context, 'project1')
api.reservation_expire(self.context)
expected = {'project_id': 'project1',
'gigabytes': {'reserved': 0, 'in_use': 0},
'volumes': {'reserved': 0, 'in_use': 0}}
self.assertEqual(expected,
api.quota_usage_get_all_by_project(
self.context,
'project1'))
class DBAPIQuotaClassTestCase(QuotaApiTestCase):
"""Tests for api.api.quota_class_* methods."""
def setUp(self):
super(DBAPIQuotaClassTestCase, self).setUp()
self.sample_qc = api.quota_class_create(self.context, 'test_qc',
'test_resource', 42)
def test_quota_class_get(self):
qc = api.quota_class_get(self.context, 'test_qc', 'test_resource')
self._assertEqualObjects(self.sample_qc, qc)
def test_quota_class_destroy(self):
api.quota_class_destroy(self.context, 'test_qc', 'test_resource')
self.assertRaises(exceptions.QuotaClassNotFound,
api.quota_class_get, self.context,
'test_qc', 'test_resource')
def test_quota_class_get_not_found(self):
self.assertRaises(exceptions.QuotaClassNotFound,
api.quota_class_get, self.context, 'nonexistent',
'nonexistent')
def test_quota_class_get_all_by_name(self):
api.quota_class_create(self.context, 'test2', 'res1', 43)
api.quota_class_create(self.context, 'test2', 'res2', 44)
self.assertEqual({'class_name': 'test_qc', 'test_resource': 42},
api.quota_class_get_all_by_name(self.context,
'test_qc'))
self.assertEqual({'class_name': 'test2', 'res1': 43, 'res2': 44},
api.quota_class_get_all_by_name(self.context,
'test2'))
def test_quota_class_update(self):
api.quota_class_update(self.context, 'test_qc', 'test_resource', 43)
updated = api.quota_class_get(self.context, 'test_qc',
'test_resource')
self.assertEqual(43, updated['hard_limit'])
def test_quota_class_destroy_all_by_name(self):
api.quota_class_create(self.context, 'test2', 'res1', 43)
api.quota_class_create(self.context, 'test2', 'res2', 44)
api.quota_class_destroy_all_by_name(self.context, 'test2')
self.assertEqual({'class_name': 'test2'},
api.quota_class_get_all_by_name(self.context,
'test2'))
class DBAPIQuotaTestCase(QuotaApiTestCase):
"""Tests for api.api.reservation_* methods."""
def test_quota_create(self):
_quota = api.quota_create(self.context, 'project1', 'resource', 99)
self.assertEqual('resource', _quota.resource)
self.assertEqual(99, _quota.hard_limit)
self.assertEqual('project1', _quota.project_id)
def test_quota_get(self):
_quota = api.quota_create(self.context, 'project1', 'resource', 99)
quota_db = api.quota_get(self.context, 'project1', 'resource')
self._assertEqualObjects(_quota, quota_db)
def test_quota_get_all_by_project(self):
for i in range(3):
for j in range(3):
api.quota_create(self.context, 'proj%d' % i, 'res%d' % j, j)
for i in range(3):
quotas_db = api.quota_get_all_by_project(self.context,
'proj%d' % i)
self.assertEqual({'project_id': 'proj%d' % i,
'res0': 0,
'res1': 1,
'res2': 2}, quotas_db)
def test_quota_update(self):
api.quota_create(self.context, 'project1', 'resource1', 41)
api.quota_update(self.context, 'project1', 'resource1', 42)
_quota = api.quota_get(self.context, 'project1', 'resource1')
self.assertEqual(42, _quota.hard_limit)
self.assertEqual('resource1', _quota.resource)
self.assertEqual('project1', _quota.project_id)
def test_quota_update_nonexistent(self):
self.assertRaises(exceptions.ProjectQuotaNotFound,
api.quota_update,
self.context,
'project1',
'resource1',
42)
def test_quota_get_nonexistent(self):
self.assertRaises(exceptions.ProjectQuotaNotFound,
api.quota_get,
self.context,
'project1',
'resource1')
def test_quota_reserve(self):
reservations = self._quota_reserve(self.context, 'project1')
self.assertEqual(2, len(reservations))
quota_usage = api.quota_usage_get_all_by_project(self.context,
'project1')
self.assertEqual({'project_id': 'project1',
'gigabytes': {'reserved': 2, 'in_use': 0},
'volumes': {'reserved': 1, 'in_use': 0}},
quota_usage)
def test_quota_destroy(self):
api.quota_create(self.context, 'project1', 'resource1', 41)
self.assertIsNone(api.quota_destroy(self.context, 'project1',
'resource1'))
self.assertRaises(exceptions.ProjectQuotaNotFound, api.quota_get,
self.context, 'project1', 'resource1')
def test_quota_destroy_by_project(self):
# Create limits, reservations and usage for project
project = 'project1'
self._quota_reserve(self.context, project)
expected_usage = {'project_id': project,
'volumes': {'reserved': 1, 'in_use': 0},
'gigabytes': {'reserved': 2, 'in_use': 0}}
expected = {'project_id': project, 'gigabytes': 2, 'volumes': 1}
# Check that quotas are there
self.assertEqual(expected,
api.quota_get_all_by_project(self.context, project))
self.assertEqual(expected_usage,
api.quota_usage_get_all_by_project(self.context,
project))
# Destroy only the limits
api.quota_destroy_by_project(self.context, project)
# Confirm that limits have been removed
self.assertEqual({'project_id': project},
api.quota_get_all_by_project(self.context, project))
# But that usage and reservations are the same
self.assertEqual(expected_usage,
api.quota_usage_get_all_by_project(self.context,
project))
def test_quota_destroy_sqlalchemy_all_by_project_(self):
# Create limits, reservations and usage for project
project = 'project1'
self._quota_reserve(self.context, project)
expected_usage = {'project_id': project,
'volumes': {'reserved': 1, 'in_use': 0},
'gigabytes': {'reserved': 2, 'in_use': 0}}
expected = {'project_id': project, 'gigabytes': 2, 'volumes': 1}
expected_result = {'project_id': project}
# Check that quotas are there
self.assertEqual(expected,
api.quota_get_all_by_project(self.context, project))
self.assertEqual(expected_usage,
api.quota_usage_get_all_by_project(self.context,
project))
# Destroy all quotas using SQLAlchemy Implementation
api.quota_destroy_all_by_project(self.context, project,
only_quotas=False)
# Check that all quotas have been deleted
self.assertEqual(expected_result,
api.quota_get_all_by_project(self.context, project))
self.assertEqual(expected_result,
api.quota_usage_get_all_by_project(self.context,
project))
def test_quota_usage_get_nonexistent(self):
self.assertRaises(exceptions.QuotaUsageNotFound,
api.quota_usage_get,
self.context,
'p1',
'nonexitent_resource')
def test_quota_usage_get(self):
self._quota_reserve(self.context, 'p1')
quota_usage = api.quota_usage_get(self.context, 'p1', 'gigabytes')
expected = {'resource': 'gigabytes', 'project_id': 'p1',
'in_use': 0, 'reserved': 2, 'total': 2}
for key, value in expected.items():
self.assertEqual(value, quota_usage[key], key)
def test_quota_usage_get_all_by_project(self):
self._quota_reserve(self.context, 'p1')
expected = {'project_id': 'p1',
'volumes': {'in_use': 0, 'reserved': 1},
'gigabytes': {'in_use': 0, 'reserved': 2}}
self.assertEqual(expected, api.quota_usage_get_all_by_project(
self.context, 'p1'))

View File

@ -14,6 +14,7 @@
# under the License.
import datetime
import inspect
import unittest
@ -47,6 +48,8 @@ def _get_field_value(column):
return 1.0
elif isinstance(column.type, sql.Boolean):
return True
elif isinstance(column.type, sql.DateTime):
return datetime.datetime.utcnow()
else:
return None
@ -222,8 +225,9 @@ class ModelsTest(unittest.TestCase):
with self.context.session.begin():
core.create_resource(
self.context, model_class, create_dict)
except Exception:
self.fail('test_resources raised Exception unexpectedly')
except Exception as e:
msg = str(e)
self.fail('test_resources raised Exception unexpectedly %s' % msg)
def test_resource_routing_unique_key(self):
pod = {'pod_id': 'test_pod1_uuid',