From b713fe23243da02657ea5039e5b75d937d867047 Mon Sep 17 00:00:00 2001 From: wanghao Date: Mon, 16 Jan 2017 14:06:04 +0800 Subject: [PATCH] Quota support in Mogan(part 1) Support quota features in Mogan. This patch will introduce the basic concepts of quota, including Quota, QuotaUsage, Reservation, QuotaDriver. Change-Id: I654b3f60880d4ab817b1b457e47882be5adca831 Implements: bp quota-support --- mogan/common/exception.py | 34 ++ mogan/conf/api.py | 20 + mogan/db/api.py | 49 +++ .../91941bf1ebc9_initial_migration.py | 52 +++ mogan/db/sqlalchemy/api.py | 325 +++++++++++++- mogan/db/sqlalchemy/models.py | 65 +++ mogan/objects/quota.py | 413 ++++++++++++++++++ .../unit/db/sqlalchemy/test_migrations.py | 15 + mogan/tests/unit/db/test_quota_usages.py | 82 ++++ mogan/tests/unit/db/test_quotas.py | 114 +++++ mogan/tests/unit/db/utils.py | 32 ++ mogan/tests/unit/objects/test_objects.py | 3 +- 12 files changed, 1202 insertions(+), 2 deletions(-) create mode 100644 mogan/objects/quota.py create mode 100644 mogan/tests/unit/db/test_quota_usages.py create mode 100644 mogan/tests/unit/db/test_quotas.py diff --git a/mogan/common/exception.py b/mogan/common/exception.py index 6764586a..c04a66e4 100644 --- a/mogan/common/exception.py +++ b/mogan/common/exception.py @@ -300,4 +300,38 @@ class InstanceIsLocked(Invalid): msg_fmt = _("Instance %(instance_uuid)s is locked") +class InvalidReservationExpiration(Invalid): + message = _("Invalid reservation expiration %(expire)s.") + + +class QuotaNotFound(NotFound): + message = _("Quota %(quota_name)s could not be found.") + + +class ProjectQuotaNotFound(QuotaNotFound): + message = _("Quota for project %(project_id)s could not be found.") + + +class QuotaResourceUnknown(QuotaNotFound): + message = _("Unknown quota resources %(unknown)s.") + + +class OverQuota(MoganException): + message = _("Quota exceeded for resources: %(overs)s") + + +class QuotaAlreadyExists(MoganException): + _msg_fmt = _("Quota with name %(name)s and project %(project_id)s already" + " exists.") + + +class ReservationAlreadyExists(MoganException): + _msg_fmt = _("Reservation with name %(name)s and project %(project_id)s " + "already exists.") + + +class ReservationNotFound(NotFound): + message = _("Reservation %(uuid)s could not be found.") + + ObjectActionError = obj_exc.ObjectActionError diff --git a/mogan/conf/api.py b/mogan/conf/api.py index a9595adf..d21416ab 100644 --- a/mogan/conf/api.py +++ b/mogan/conf/api.py @@ -62,7 +62,27 @@ opts = [ opt_group = cfg.OptGroup(name='api', title='Options for the mogan-api service') +quota_opts = [ + cfg.StrOpt('quota_driver', + help=_("Specify the quota driver which is used in Mogan " + "service.")), + cfg.IntOpt('reservation_expire', + default=86400, + help=_('Number of seconds until a reservation expires')), + cfg.IntOpt('until_refresh', + default=0, + help=_('Count of reservations until usage is refreshed')), + cfg.IntOpt('max_age', + default=0, + help=_('Number of seconds between subsequent usage refreshes')), +] + +opt_quota_group = cfg.OptGroup(name='quota', + title='Options for the mogan quota') + def register_opts(conf): conf.register_group(opt_group) conf.register_opts(opts, group=opt_group) + conf.register_group(opt_quota_group) + conf.register_opts(quota_opts, group=opt_quota_group) diff --git a/mogan/db/api.py b/mogan/db/api.py index fca6fcfb..417a72c5 100644 --- a/mogan/db/api.py +++ b/mogan/db/api.py @@ -123,3 +123,52 @@ class Connection(object): @abc.abstractmethod def instance_fault_get_by_instance_uuids(self, context, instance_uuids): """Get all instance faults for the provided instance_uuids.""" + + @abc.abstractmethod + def quota_get(self, context, project_id, resource_name): + """Get quota value of a resource""" + + @abc.abstractmethod + def quota_get_all(self, context, project_only=False): + """Get all quotas value of resources""" + + @abc.abstractmethod + def quota_create(self, context, values): + """Create a quota of a resource""" + + @abc.abstractmethod + def quota_destroy(self, context, project_id, resource_name): + """Delete a quota of a resource""" + + @abc.abstractmethod + def quota_update(self, context, project_id, resource_name, updates): + """Delete a quota of a resource""" + + @abc.abstractmethod + def quota_get_all_by_project(self, context, project_id): + """Get quota by project id""" + + @abc.abstractmethod + def quota_usage_get_all_by_project(self, context, project_id): + """Get quota usage by project id""" + + @abc.abstractmethod + def quota_allocated_get_all_by_project(self, context, project_id): + """Get quota usage by project id""" + + @abc.abstractmethod + def quota_reserve(self, context, resources, quotas, deltas, expire, + until_refresh, max_age, project_id): + """Reserve quota of resource""" + + @abc.abstractmethod + def reservation_commit(self, context, reservations, project_id): + """Commit reservation of quota usage""" + + @abc.abstractmethod + def reservation_rollback(self, context, reservations, project_id): + """Reservation rollback""" + + @abc.abstractmethod + def reservation_expire(self, context): + """expire all reservations which has been expired""" diff --git a/mogan/db/sqlalchemy/alembic/versions/91941bf1ebc9_initial_migration.py b/mogan/db/sqlalchemy/alembic/versions/91941bf1ebc9_initial_migration.py index 19717f65..1bf3055c 100644 --- a/mogan/db/sqlalchemy/alembic/versions/91941bf1ebc9_initial_migration.py +++ b/mogan/db/sqlalchemy/alembic/versions/91941bf1ebc9_initial_migration.py @@ -124,3 +124,55 @@ def upgrade(): mysql_ENGINE='InnoDB', mysql_DEFAULT_CHARSET='UTF8' ) + op.create_table( + 'quotas', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.String(length=36), nullable=True), + sa.Column('resource_name', sa.String(length=255), nullable=True), + sa.Column('hard_limit', sa.Integer(), nullable=True), + sa.Column('allocated', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('resource_name', 'project_id', + name='uniq_quotas0resource_name'), + mysql_ENGINE='InnoDB', + mysql_DEFAULT_CHARSET='UTF8' + ) + op.create_table( + 'quota_usages', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.String(length=36), nullable=True), + sa.Column('resource_name', sa.String(length=255), nullable=True), + sa.Column('in_use', sa.Integer(), nullable=True), + sa.Column('reserved', sa.Integer(), nullable=True), + sa.Column('until_refresh', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('resource_name', 'project_id', + name='uniq_quotas0resource_name'), + mysql_ENGINE='InnoDB', + mysql_DEFAULT_CHARSET='UTF8' + ) + op.create_table( + 'reservations', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', sa.String(length=36), nullable=True), + sa.Column('usage_id', sa.Integer(), nullable=False), + sa.Column('allocated_id', sa.Integer(), nullable=False), + sa.Column('project_id', sa.String(length=36), nullable=True), + sa.Column('resource_name', sa.String(length=255), nullable=True), + sa.Column('delta', sa.Integer(), nullable=True), + sa.Column('expire', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['usage_id'], + ['quota_usages.id']), + sa.ForeignKeyConstraint(['allocated_id'], + ['quotas.id']), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('uuid', name='uniq_reservation0uuid'), + mysql_ENGINE='InnoDB', + mysql_DEFAULT_CHARSET='UTF8' + ) diff --git a/mogan/db/sqlalchemy/api.py b/mogan/db/sqlalchemy/api.py index b9e90f51..a4cfb3aa 100644 --- a/mogan/db/sqlalchemy/api.py +++ b/mogan/db/sqlalchemy/api.py @@ -20,19 +20,22 @@ import threading from oslo_db import exception as db_exc from oslo_db.sqlalchemy import enginefacade from oslo_db.sqlalchemy import utils as sqlalchemyutils +from oslo_log import log as logging from oslo_utils import strutils +from oslo_utils import timeutils from oslo_utils import uuidutils from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.orm import joinedload from sqlalchemy.sql.expression import desc from mogan.common import exception -from mogan.common.i18n import _ +from mogan.common.i18n import _, _LW from mogan.db import api from mogan.db.sqlalchemy import models _CONTEXT = threading.local() +LOG = logging.getLogger(__name__) def get_backend(): @@ -109,6 +112,7 @@ class Connection(api.Connection): """SqlAlchemy connection.""" def __init__(self): + self.QUOTA_SYNC_FUNCTIONS = {'_sync_instances': self._sync_instances} pass def instance_type_create(self, context, values): @@ -346,6 +350,325 @@ class Connection(api.Connection): return output + def quota_get(self, context, project_id, resource_name): + query = model_query( + context, + models.Quota).filter_by(project_id=project_id, + resource_name=resource_name) + try: + return query.one() + except NoResultFound: + raise exception.QuotaNotFound(quota_name=resource_name) + + def quota_create(self, context, values): + quota = models.Quota() + quota.update(values) + + with _session_for_write() as session: + try: + session.add(quota) + session.flush() + except db_exc.DBDuplicateEntry: + project_id = values['project_id'] + raise exception.QuotaAlreadyExists(name=values['name'], + project_id=project_id) + return quota + + def quota_get_all(self, context, project_only): + return model_query(context, models.Quota, project_only=project_only) + + def quota_destroy(self, context, project_id, resource_name): + with _session_for_write(): + query = model_query(context, models.Quota) + query = query.filter_by(project_id=project_id, + resource_name=resource_name) + + count = query.delete() + if count != 1: + raise exception.QuotaNotFound(quota_name=resource_name) + + def _do_update_quota(self, context, project_id, resource_name, updates): + with _session_for_write(): + query = model_query(context, models.Quota) + query = query.filter_by(project_id=project_id, + resource_name=resource_name) + try: + ref = query.with_lockmode('update').one() + except NoResultFound: + raise exception.QuotaNotFound(quota_name=resource_name) + + ref.update(updates) + return ref + + def quota_update(self, context, project_id, resource_name, updates): + if 'resource_name' in updates or 'project_id' in updates: + msg = _("Cannot overwrite resource_name/project_id for " + "an existing Quota.") + raise exception.InvalidParameterValue(err=msg) + try: + return self._do_update_quota(context, project_id, resource_name, + updates) + except db_exc.DBDuplicateEntry: + pass + + def quota_get_all_by_project(self, context, project_id): + return model_query(context, models.Quota, project_id=project_id) + + def quota_usage_get_all_by_project(self, context, project_id): + rows = model_query(context, models.QuotaUsage, + project_id=project_id) + result = {'project_id': project_id} + for row in rows: + result[row.resource_name] = dict(in_use=row.in_use, + reserved=row.reserved) + return result + + def quota_allocated_get_all_by_project(self, context, project_id): + rows = model_query(context, models.Quota, + project_id=project_id) + result = {'project_id': project_id} + for row in rows: + result[row.resource_name] = row.allocated + return result + + def _get_quota_usages(self, context, project_id): + # Broken out for testability + rows = model_query(context, models.QuotaUsage, + project_id=project_id).\ + order_by(models.QuotaUsage.id.asc()).\ + with_lockmode('update').all() + return {row.resource_name: row for row in rows} + + def quota_allocated_update(self, context, project_id, resource, allocated): + with _session_for_write(): + quota_ref = self.quota_get(context, project_id, resource) + quota_ref.update({'allocated': allocated}) + return quota_ref + + def _quota_usage_create(self, context, project_id, resource, in_use, + reserved, until_refresh, session=None): + quota_usage_ref = models.QuotaUsage() + quota_usage_ref.project_id = project_id + quota_usage_ref.resource_name = resource + quota_usage_ref.in_use = in_use + quota_usage_ref.reserved = reserved + quota_usage_ref.until_refresh = until_refresh + try: + session.add(quota_usage_ref) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.QuotaAlreadyExists(name=resource, + project_id=project_id) + return quota_usage_ref + + def _reservation_create(self, context, uuid, usage, project_id, resource, + delta, expire, session=None, allocated_id=None): + usage_id = usage['id'] if usage else None + reservation_ref = models.Reservation() + reservation_ref.uuid = uuid + reservation_ref.usage_id = usage_id + reservation_ref.project_id = project_id + reservation_ref.resource_name = resource + reservation_ref.delta = delta + reservation_ref.expire = expire + reservation_ref.allocated_id = allocated_id + try: + session.add(reservation_ref) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.ReservationAlreadyExists(name=resource, + project_id=project_id) + return reservation_ref + + def _sync_instances(self, context, project_id): + query = model_query(context, models.Instance, instance=True).\ + filter_by(project_id=project_id).all() + return {'instances': len(query) or 0} + + def quota_reserve(self, context, resources, quotas, deltas, expire, + until_refresh, max_age, project_id, + is_allocated_reserve=False): + # NOTE(wanghao): Now we still doesn't support contenxt.elevated() yet. + # We can support it later. + elevated = context + with _session_for_write() as session: + if project_id is None: + project_id = context.project_id + # Get the current usages + usages = self._get_quota_usages(context, project_id) + allocated = self.quota_allocated_get_all_by_project(context, + project_id) + allocated.pop('project_id') + + # Handle usage refresh + work = set(deltas.keys()) + while work: + resource = work.pop() + + # Do we need to refresh the usage? + refresh = False + if resource not in usages: + usages[resource] = self._quota_usage_create( + elevated, project_id, resource, 0, 0, + until_refresh or None, session=session) + refresh = True + elif usages[resource].in_use < 0: + refresh = True + elif usages[resource].until_refresh is not None: + usages[resource].until_refresh -= 1 + if usages[resource].until_refresh <= 0: + refresh = True + elif max_age and usages[resource].updated_at is not None and ( + (usages[resource].updated_at - + timeutils.utcnow()).seconds >= max_age): + refresh = True + + # OK, refresh the usage + if refresh: + # Grab the sync routine + sync = self.QUOTA_SYNC_FUNCTIONS[resources[resource].sync] + updates = sync(elevated, project_id) + for res, in_use in updates.items(): + # Make sure we have a destination for the usage! + if res not in usages: + usages[res] = self._quota_usage_create( + elevated, project_id, res, 0, 0, + until_refresh or None, session=session) + + # Update the usage + usages[res].in_use = in_use + usages[res].until_refresh = until_refresh or None + + # Because more than one resource may be refreshed + # by the call to the sync routine, and we don't + # want to double-sync, we make sure all refreshed + # resources are dropped from the work set. + work.discard(res) + + # Check for deltas that would go negative + if is_allocated_reserve: + unders = [r for r, delta in deltas.items() + if delta < 0 and delta + allocated.get(r, 0) < 0] + else: + unders = [r for r, delta in deltas.items() + if delta < 0 and delta + usages[r].in_use < 0] + + # Now, let's check the quotas + overs = [r for r, delta in deltas.items() + if quotas[r] >= 0 and delta >= 0 and + quotas[r] < delta + usages[r].total + allocated.get(r, 0)] + + # Create the reservations + if not overs: + reservations = [] + for resource, delta in deltas.items(): + usage = usages[resource] + allocated_id = None + if is_allocated_reserve: + try: + quota = self.quota_get(context, project_id, + resource) + except exception.ProjectQuotaNotFound: + # If we were using the default quota, create DB + # entry + quota = self.quota_create(context, + project_id, + resource, + quotas[resource], 0) + # Since there's no reserved/total for allocated, update + # allocated immediately and subtract on rollback + # if needed + self.quota_allocated_update(context, project_id, + resource, + quota.allocated + delta) + allocated_id = quota.id + usage = None + reservation = self._reservation_create( + elevated, uuidutils.generate_uuid(), usage, project_id, + resource, delta, expire, session=session, + allocated_id=allocated_id) + + reservations.append(reservation) + + # Also update the reserved quantity + if delta > 0 and not is_allocated_reserve: + usages[resource].reserved += delta + + if unders: + LOG.warning(_LW("Change will make usage less than 0 for the " + "following resources: %s"), unders) + if overs: + usages = {k: dict(in_use=v.in_use, reserved=v.reserved, + allocated=allocated.get(k, 0)) + for k, v in usages.items()} + raise exception.OverQuota(overs=sorted(overs), quotas=quotas, + usages=usages) + return reservations + + def _dict_with_usage_id(self, usages): + return {row.id: row for row in usages.values()} + + def reservation_commit(self, context, reservations, project_id): + with _session_for_write(): + usages = self._get_quota_usages(context, project_id) + usages = self._dict_with_usage_id(usages) + + for reservation in reservations: + # Allocated reservations will have already been bumped + if not reservation.allocated_id: + usage = usages[reservation.usage_id] + if reservation.delta >= 0: + usage.reserved -= reservation.delta + usage.in_use += reservation.delta + + query = model_query(context, models.Reservation) + query = query.filter_by(uuid=reservation.uuid) + count = query.delete() + if count != 1: + raise exception.ReservationNotFound(uuid=reservation.uuid) + + def reservation_rollback(self, context, reservations, project_id): + with _session_for_write(): + usages = self._get_quota_usages(context, project_id) + usages = self._dict_with_usage_id(usages) + for reservation in reservations: + if reservation.allocated_id: + reservation.quota.allocated -= reservation.delta + else: + usage = usages[reservation.usage_id] + if reservation.delta >= 0: + usage.reserved -= reservation.delta + + query = model_query(context, models.Reservation) + query = query.filter_by(uuid=reservation.uuid) + count = query.delete() + if count != 1: + raise exception.ReservationNotFound(uuid=reservation.uuid) + + def reservation_expire(self, context): + with _session_for_write() as session: + current_time = timeutils.utcnow() + results = model_query(context, models.Reservation).\ + filter(models.Reservation.expire < current_time).\ + all() + + if results: + for reservation in results: + if reservation.delta >= 0: + if reservation.allocated_id: + reservation.quota.allocated -= reservation.delta + reservation.quota.save(session=session) + else: + reservation.usage.reserved -= reservation.delta + reservation.usage.save(session=session) + + query = model_query(context, models.Reservation) + query = query.filter_by(uuid=reservation.uuid) + count = query.delete() + if count != 1: + uuid = reservation.uuid + raise exception.ReservationNotFound(uuid=uuid) + def _type_get_id_from_type_query(context, type_id): return model_query(context, models.InstanceTypes). \ diff --git a/mogan/db/sqlalchemy/models.py b/mogan/db/sqlalchemy/models.py index 68db2e33..59561ea9 100644 --- a/mogan/db/sqlalchemy/models.py +++ b/mogan/db/sqlalchemy/models.py @@ -186,3 +186,68 @@ class InstanceFault(Base): backref=orm.backref('instance_faults', uselist=False), foreign_keys=instance_uuid, primaryjoin='Instance.uuid == InstanceFault.instance_uuid') + + +class Quota(Base): + """Represents a single quota override for a project.""" + + __tablename__ = 'quotas' + __table_args__ = ( + schema.UniqueConstraint('resource_name', 'project_id', + name='uniq_quotas0resource_name'), + table_args() + ) + id = Column(Integer, primary_key=True) + resource_name = Column(String(255), nullable=False) + project_id = Column(String(36), nullable=False) + hard_limit = Column(Integer, nullable=False) + allocated = Column(Integer, default=0) + + +class QuotaUsage(Base): + """Represents the current usage for a given resource.""" + + __tablename__ = 'quota_usages' + __table_args__ = ( + schema.UniqueConstraint('resource_name', 'project_id', + name='uniq_quotas0resource_name'), + table_args() + ) + + id = Column(Integer, primary_key=True) + project_id = Column(String(255), index=True) + resource_name = Column(String(255)) + in_use = Column(Integer) + reserved = Column(Integer) + until_refresh = Column(Integer, nullable=True) + + @property + def total(self): + return self.in_use + self.reserved + + +class Reservation(Base): + """Represents a resource reservation for quotas.""" + + __tablename__ = 'reservations' + __table_args__ = ( + schema.UniqueConstraint('uuid', name='uniq_reservation0uuid'), + table_args() + ) + + id = Column(Integer, primary_key=True) + uuid = Column(String(36), nullable=False) + usage_id = Column(Integer, ForeignKey('quota_usages.id'), nullable=True) + allocated_id = Column(Integer, ForeignKey('quotas.id'), nullable=True) + project_id = Column(String(255), index=True) + resource_name = Column(String(255)) + delta = Column(Integer) + expire = Column(DateTime, nullable=False) + + usage = orm.relationship( + "QuotaUsage", foreign_keys=usage_id, + primaryjoin='Reservation.usage_id == QuotaUsage.id') + + quota = orm.relationship( + "Quota", foreign_keys=allocated_id, + primaryjoin='Reservation.allocated_id == Quota.id') diff --git a/mogan/objects/quota.py b/mogan/objects/quota.py new file mode 100644 index 00000000..4395bf44 --- /dev/null +++ b/mogan/objects/quota.py @@ -0,0 +1,413 @@ +# Copyright 2017 Fiberhome Integration Technologies Co.,LTD +# All Rights Reserved. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Quotas for instances.""" + +import datetime + +from oslo_config import cfg +from oslo_utils import importutils +from oslo_utils import timeutils +from oslo_versionedobjects import base as object_base +import six + +from mogan.common import exception +from mogan.db import api as dbapi +from mogan.objects import base +from mogan.objects import fields as object_fields + + +CONF = cfg.CONF + + +@base.MoganObjectRegistry.register +class Quota(base.MoganObject, object_base.VersionedObjectDictCompat): + # Version 1.0: Initial version + VERSION = '1.0' + + dbapi = dbapi.get_instance() + + fields = { + 'id': object_fields.IntegerField(), + 'project_id': object_fields.UUIDField(nullable=True), + 'resource_name': object_fields.StringField(nullable=True), + 'hard_limit': object_fields.IntegerField(nullable=True), + 'allocated': object_fields.IntegerField(default=0), + } + + def __init__(self, *args, **kwargs): + super(Quota, self).__init__(*args, **kwargs) + self.quota_driver = importutils.import_object(CONF.quota.quota_driver) + self._resources = {} + + @property + def resources(self): + return self._resources + + @staticmethod + def _from_db_object_list(db_objects, cls, context): + """Converts a list of database entities to a list of formal objects.""" + return [Quota._from_db_object(cls(context), obj) + for obj in db_objects] + + @classmethod + def list(cls, context, project_only=False): + """Return a list of Quota objects.""" + db_quotas = cls.dbapi.quota_get_all(context, + project_only=project_only) + return Quota._from_db_object_list(db_quotas, cls, context) + + @classmethod + def get(cls, context, project_id, resource_name): + """Find a quota of resource and return a Quota object.""" + db_quota = cls.dbapi.quota_get(context, project_id, resource_name) + quota = Quota._from_db_object(cls(context), db_quota) + return quota + + def create(self, context): + """Create a Quota record in the DB.""" + values = self.obj_get_changes() + # Since we need to avoid passing False down to the DB layer + # (which uses an integer), we can always default it to zero here. + values['deleted'] = 0 + + db_quota = self.dbapi.quota_create(context, values) + self._from_db_object(self, db_quota) + + def destroy(self, context, project_id, resource_name): + """Delete the Quota from the DB.""" + self.dbapi.quota_destroy(context, project_id, resource_name) + self.obj_reset_changes() + + def save(self, context, project_id, resource_name): + """Save updates to this Quota.""" + updates = self.obj_get_changes() + self.dbapi.quota_update(context, project_id, resource_name, updates) + self.obj_reset_changes() + + def refresh(self, context, project_id, resource_name): + """Refresh the object by re-fetching from the DB.""" + current = self.__class__.get(context, project_id, resource_name) + self.obj_refresh(current) + self.obj_reset_changes() + + def reserve(self, context, expire=None, project_id=None, **deltas): + """reserve the Quota.""" + return self.quota_driver.reserver(context, self.resources, deltas, + expire=expire, + project_id=project_id) + + def commit(self, context, reservations, project_id=None): + self.quota_driver.commit(context, reservations, project_id=project_id) + + def rollback(self, context, reservations, project_id=None): + self.quota_driver.rollback(context, reservations, + project_id=project_id) + + def expire(self, context): + return self.quota_driver.expire(context) + + def count(self, context, resource, *args, **kwargs): + """Count a resource. + + For countable resources, invokes the count() function and + returns its result. Arguments following the context and + resource are passed directly to the count function declared by + the resource. + + :param context: The request context, for access checks. + :param resource: The name of the resource, as a string. + """ + + # Get the resource + res = self.resources.get(resource) + if not res or not hasattr(res, 'count'): + raise exception.QuotaResourceUnknown(unknown=[resource]) + + return res.count(context, *args, **kwargs) + + def register_resource(self, resource): + """Register a resource.""" + + self._resources[resource.name] = resource + + def register_resources(self, resources): + """Register a list of resources.""" + + for resource in resources: + self.register_resource(resource) + + +class DbQuotaDriver(object): + + """Driver to perform check to enforcement of quotas. + + Also allows to obtain quota information. + The default driver utilizes the local database. + """ + + def get_project_quotas(self, context, resources, project_id, + quota_class=None, defaults=True, + usages=True): + """Retrieve quotas for a project. + + Given a list of resources, retrieve the quotas for the given + project. + + :param context: The request context, for access checks. + :param resources: A dictionary of the registered resources. + :param project_id: The ID of the project to return quotas for. + :param quota_class: If project_id != context.tenant, the + quota class cannot be determined. This + parameter allows it to be specified. It + will be ignored if project_id == + context.tenant. + :param defaults: If True, the quota class value (or the + default value, if there is no value from the + quota class) will be reported if there is no + specific value for the resource. + :param usages: If True, the current in_use, reserved and allocated + counts will also be returned. + """ + + quotas = {} + project_quotas = dbapi.quota_get_all_by_project(context, project_id) + allocated_quotas = None + if usages: + project_usages = dbapi.quota_usage_get_all_by_project(context, + project_id) + allocated_quotas = dbapi.quota_allocated_get_all_by_project( + context, project_id) + allocated_quotas.pop('project_id') + + for resource in resources.values(): + # Omit default/quota class values + if not defaults and resource.name not in project_quotas: + continue + + quota_val = project_quotas.get(resource.name) + if quota_val is None: + raise exception.QuotaNotFound(quota_name=resource.name) + quotas[resource.name] = {'limit': quota_val} + + # Include usages if desired. This is optional because one + # internal consumer of this interface wants to access the + # usages directly from inside a transaction. + if usages: + usage = project_usages.get(resource.name, {}) + quotas[resource.name].update( + in_use=usage.get('in_use', 0), + reserved=usage.get('reserved', 0), ) + if allocated_quotas: + quotas[resource.name].update( + allocated=allocated_quotas.get(resource.name, 0), ) + return quotas + + def _get_quotas(self, context, resources, keys, has_sync, project_id=None): + """A helper method which retrieves the quotas for specific resources. + + This specific resource is identified by keys, and which apply to the + current context. + + :param context: The request context, for access checks. + :param resources: A dictionary of the registered resources. + :param keys: A list of the desired quotas to retrieve. + :param has_sync: If True, indicates that the resource must + have a sync attribute; if False, indicates + that the resource must NOT have a sync + attribute. + :param project_id: Specify the project_id if current context + is admin and admin wants to impact on + common user's tenant. + """ + + # Filter resources + if has_sync: + sync_filt = lambda x: hasattr(x, 'sync') + else: + sync_filt = lambda x: not hasattr(x, 'sync') + desired = set(keys) + sub_resources = {k: v for k, v in resources.items() + if k in desired and sync_filt(v)} + + # Make sure we accounted for all of them... + if len(keys) != len(sub_resources): + unknown = desired - set(sub_resources.keys()) + raise exception.QuotaResourceUnknown(unknown=sorted(unknown)) + + # Grab and return the quotas (without usages) + quotas = self.get_project_quotas(context, sub_resources, + project_id, + context.quota_class, usages=False) + + return {k: v['limit'] for k, v in quotas.items()} + + def reserve(self, context, resources, deltas, expire=None, + project_id=None): + """Check quotas and reserve resources. + + For counting quotas--those quotas for which there is a usage + synchronization function--this method checks quotas against + current usage and the desired deltas. + + This method will raise a QuotaResourceUnknown exception if a + given resource is unknown or if it does not have a usage + synchronization function. + + If any of the proposed values is over the defined quota, an + OverQuota exception will be raised with the sorted list of the + resources which are too high. Otherwise, the method returns a + list of reservation UUIDs which were created. + + :param context: The request context, for access checks. + :param resources: A dictionary of the registered resources. + :param deltas: A dictionary of the proposed delta changes. + :param expire: An optional parameter specifying an expiration + time for the reservations. If it is a simple + number, it is interpreted as a number of + seconds and added to the current time; if it is + a datetime.timedelta object, it will also be + added to the current time. A datetime.datetime + object will be interpreted as the absolute + expiration time. If None is specified, the + default expiration time set by + --default-reservation-expire will be used (this + value will be treated as a number of seconds). + :param project_id: Specify the project_id if current context + is admin and admin wants to impact on + common user's tenant. + """ + + # Set up the reservation expiration + if expire is None: + expire = CONF.quota.reservation_expire + if isinstance(expire, six.integer_types): + expire = datetime.timedelta(seconds=expire) + if isinstance(expire, datetime.timedelta): + expire = timeutils.utcnow() + expire + if not isinstance(expire, datetime.datetime): + raise exception.InvalidReservationExpiration(expire=expire) + + # If project_id is None, then we use the project_id in context + if project_id is None: + project_id = context.tenant + + # Get the applicable quotas. + quotas = self._get_quotas(context, resources, deltas.keys(), + has_sync=True, project_id=project_id) + + return self._reserve(context, resources, quotas, deltas, expire, + project_id) + + def _reserve(self, context, resources, quotas, deltas, expire, project_id): + return dbapi.quota_reserve(context, resources, quotas, deltas, expire, + CONF.quota.until_refresh, + CONF.quota.max_age, + project_id=project_id) + + def commit(self, context, reservations, project_id=None): + """Commit reservations. + + :param context: The request context, for access checks. + :param reservations: A list of the reservation UUIDs, as + returned by the reserve() method. + :param project_id: Specify the project_id if current context + is admin and admin wants to impact on + common user's tenant. + """ + # If project_id is None, then we use the project_id in context + if project_id is None: + project_id = context.tenant + + dbapi.reservation_commit(context, reservations, project_id=project_id) + + def rollback(self, context, reservations, project_id=None): + """Roll back reservations. + + :param context: The request context, for access checks. + :param reservations: A list of the reservation UUIDs, as + returned by the reserve() method. + :param project_id: Specify the project_id if current context + is admin and admin wants to impact on + common user's tenant. + """ + # If project_id is None, then we use the project_id in context + if project_id is None: + project_id = context.tenant + + dbapi.reservation_rollback(context, reservations, + project_id=project_id) + + def expire(self, context): + """Expire reservations. + + Explores all currently existing reservations and rolls back + any that have expired. + + :param context: The request context, for access checks. + """ + + dbapi.reservation_expire(context) + + +class BaseResource(object): + """Describe a single resource for quota checking.""" + + def __init__(self, name, sync, count=None): + """Initializes a Resource. + + :param name: The name of the resource, i.e., "instances". + :param sync: A dbapi methods name which returns a dictionary + to resynchronize the in_use count for one or more + resources, as described above. + """ + self.name = name + self.sync = sync + self.count = count + + def quota(self, driver, context, **kwargs): + """Given a driver and context, obtain the quota for this resource. + + :param driver: A quota driver. + :param context: The request context. + :param project_id: The project to obtain the quota value for. + If not provided, it is taken from the + context. If it is given as None, no + project-specific quota will be searched + for. + """ + + # Get the project ID + project_id = kwargs.get('project_id', context.tenant) + + # Look up the quota for the project + if project_id: + try: + return driver.get_by_project(context, project_id, self.name) + except exception.ProjectQuotaNotFound: + pass + return -1 + + +class InstanceResource(BaseResource): + """ReservableResource for a specific instance.""" + + def __init__(self, name='instances'): + """Initializes a InstanceResource. + + :param name: The kind of resource, i.e., "instances". + """ + super(InstanceResource, self).__init__(name, "_sync_%s" % name) diff --git a/mogan/tests/unit/db/sqlalchemy/test_migrations.py b/mogan/tests/unit/db/sqlalchemy/test_migrations.py index 0cf5c26d..deed515d 100644 --- a/mogan/tests/unit/db/sqlalchemy/test_migrations.py +++ b/mogan/tests/unit/db/sqlalchemy/test_migrations.py @@ -230,6 +230,21 @@ class MigrationCheckersMixin(object): self.assertIn('created_at', col_names) self.assertIsInstance(nodes.c.provision_updated_at.type, sqlalchemy.types.DateTime) + nodes = db_utils.get_table(engine, 'quotas') + col_names = [column.name for column in nodes.c] + self.assertIn('created_at', col_names) + self.assertIsInstance(nodes.c.resource_name.type, + sqlalchemy.types.String) + nodes = db_utils.get_table(engine, 'quota_usages') + col_names = [column.name for column in nodes.c] + self.assertIn('created_at', col_names) + self.assertIsInstance(nodes.c.resource_name.type, + sqlalchemy.types.String) + nodes = db_utils.get_table(engine, 'reservations') + col_names = [column.name for column in nodes.c] + self.assertIn('created_at', col_names) + self.assertIsInstance(nodes.c.resource_name.type, + sqlalchemy.types.String) def test_upgrade_and_version(self): with patch_with_engine(self.engine): diff --git a/mogan/tests/unit/db/test_quota_usages.py b/mogan/tests/unit/db/test_quota_usages.py new file mode 100644 index 00000000..dffebc79 --- /dev/null +++ b/mogan/tests/unit/db/test_quota_usages.py @@ -0,0 +1,82 @@ +# Copyright 2018 FiberHome Telecommunication Technologies CO.,LTD +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests for manipulating QuotaUsages via the DB API""" + +import datetime + +from oslo_config import cfg +from oslo_context import context + +from mogan.db import api as db_api +from mogan.objects import quota +from mogan.tests.unit.db import base +from mogan.tests.unit.db import utils + + +CONF = cfg.CONF + + +class DbQuotaUsageTestCase(base.DbTestCase): + + def setUp(self): + super(DbQuotaUsageTestCase, self).setUp() + self.context = context.get_admin_context() + self.instance = quota.InstanceResource() + self.resources = {self.instance.name: self.instance} + self.project_id = "c18e8a1a870d4c08a0b51ced6e0b6459" + + def test_quota_usage_reserve(self): + utils.create_test_quota() + dbapi = db_api.get_instance() + r = dbapi.quota_reserve(self.context, self.resources, + {'instances': 10}, + {'instances': 1}, + datetime.datetime(2099, 1, 1, 0, 0), + CONF.quota.until_refresh, CONF.quota.max_age, + project_id=self.project_id) + self.assertEqual('instances', r[0].resource_name) + + def test_reserve_commit(self): + utils.create_test_quota() + dbapi = db_api.get_instance() + rs = dbapi.quota_reserve(self.context, self.resources, + {'instances': 10}, + {'instances': 1}, + datetime.datetime(2099, 1, 1, 0, 0), + CONF.quota.until_refresh, CONF.quota.max_age, + project_id=self.project_id) + r = dbapi.quota_usage_get_all_by_project(self.context, self.project_id) + before_in_use = r['instances']['in_use'] + dbapi.reservation_commit(self.context, rs, self.project_id) + r = dbapi.quota_usage_get_all_by_project(self.context, self.project_id) + after_in_use = r['instances']['in_use'] + self.assertEqual(before_in_use + 1, after_in_use) + + def test_reserve_rollback(self): + utils.create_test_quota() + dbapi = db_api.get_instance() + rs = dbapi.quota_reserve(self.context, self.resources, + {'instances': 10}, + {'instances': 1}, + datetime.datetime(2099, 1, 1, 0, 0), + CONF.quota.until_refresh, CONF.quota.max_age, + project_id=self.project_id) + r = dbapi.quota_usage_get_all_by_project(self.context, self.project_id) + before_in_use = r['instances']['in_use'] + dbapi.reservation_rollback(self.context, rs, self.project_id) + r = dbapi.quota_usage_get_all_by_project(self.context, self.project_id) + after_in_use = r['instances']['in_use'] + self.assertEqual(before_in_use, after_in_use) diff --git a/mogan/tests/unit/db/test_quotas.py b/mogan/tests/unit/db/test_quotas.py new file mode 100644 index 00000000..eb8c255b --- /dev/null +++ b/mogan/tests/unit/db/test_quotas.py @@ -0,0 +1,114 @@ +# Copyright 2018 FiberHome Telecommunication Technologies CO.,LTD +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests for manipulating Quotas via the DB API""" + +import six + +from mogan.common import exception +from mogan.tests.unit.db import base +from mogan.tests.unit.db import utils + + +class DbQuotaTestCase(base.DbTestCase): + + def test_quota_create(self): + utils.create_test_quota() + + def test_quota_get_by_project_id_and_resource_name(self): + quota = utils.create_test_quota() + res = self.dbapi.quota_get(self.context, quota.project_id, + quota.resource_name) + self.assertEqual(quota.id, res.id) + + def test_quota_get_not_exist(self): + self.assertRaises(exception.QuotaNotFound, + self.dbapi.quota_get, + self.context, + '123', 'fake') + + def test_quota_get_all(self): + ids_project_1 = [] + ids_project_2 = [] + ids_project_all = [] + resource_names = ['instances', 'instances_type', 'test_resource'] + for i in range(0, 3): + quota = utils.create_test_quota(project_id='project_1', + resource_name=resource_names[i]) + ids_project_1.append(quota['id']) + for i in range(3, 5): + resource_name = resource_names[i - 3] + quota = utils.create_test_quota(project_id='project_2', + resource_name=resource_name) + ids_project_2.append(quota['id']) + ids_project_all.extend(ids_project_1) + ids_project_all.extend(ids_project_2) + + # Set project_only to False + # get all quotas from all projects + res = self.dbapi.quota_get_all(self.context, project_only=False) + res_ids = [r.id for r in res] + six.assertCountEqual(self, ids_project_all, res_ids) + + # Set project_only to True + # get quotas from current project (project_1) + self.context.tenant = 'project_1' + res = self.dbapi.quota_get_all(self.context, project_only=True) + res_ids = [r.id for r in res] + six.assertCountEqual(self, ids_project_1, res_ids) + + # Set project_only to True + # get quotas from current project (project_2) + self.context.tenant = 'project_2' + res = self.dbapi.quota_get_all(self.context, project_only=True) + res_ids = [r.id for r in res] + six.assertCountEqual(self, ids_project_2, res_ids) + + def test_quota_destroy(self): + quota = utils.create_test_quota() + self.dbapi.quota_destroy(self.context, quota.project_id, + quota.resource_name) + self.assertRaises(exception.QuotaNotFound, + self.dbapi.quota_get, + self.context, + quota.project_id, + quota.resource_name) + + def test_quota_destroy_not_exist(self): + self.assertRaises(exception.QuotaNotFound, + self.dbapi.quota_destroy, + self.context, + '123', 'fake') + + def test_quota_update(self): + quota = utils.create_test_quota() + old_limit = quota.hard_limit + new_limit = 100 + self.assertNotEqual(old_limit, new_limit) + + res = self.dbapi.quota_update(self.context, + quota.project_id, + quota.resource_name, + {'hard_limit': new_limit}) + self.assertEqual(new_limit, res.hard_limit) + + def test_quota_update_with_invalid_parameter_value(self): + quota = utils.create_test_quota() + self.assertRaises(exception.InvalidParameterValue, + self.dbapi.quota_update, + self.context, + quota.project_id, + quota.resource_name, + {'resource_name': 'instance_test'}) diff --git a/mogan/tests/unit/db/utils.py b/mogan/tests/unit/db/utils.py index 49fdf322..503acdb8 100644 --- a/mogan/tests/unit/db/utils.py +++ b/mogan/tests/unit/db/utils.py @@ -68,6 +68,19 @@ def get_test_instance(**kw): } +def get_test_quota(**kw): + return { + 'id': kw.get('id', 123), + 'resource_name': kw.get('resource_name', 'instances'), + 'project_id': kw.get('project_id', + 'c18e8a1a870d4c08a0b51ced6e0b6459'), + 'hard_limit': kw.get('hard_limit', 10), + 'allocated': kw.get('allocated', 0), + 'created_at': kw.get('created_at'), + 'updated_at': kw.get('updated_at'), + } + + def create_test_instance(context={}, **kw): """Create test instance entry in DB and return Instance DB object. @@ -112,3 +125,22 @@ def create_test_instance_type(context={}, **kw): dbapi = db_api.get_instance() return dbapi.instance_type_create(context, instance_type) + + +def create_test_quota(context={}, **kw): + """Create test quota entry in DB and return quota DB object. + + Function to be used to create test Quota objects in the database. + + :param context: The request context, for access checks. + :param kw: kwargs with overriding values for instance's attributes. + :returns: Test Quota DB object. + + """ + quota = get_test_quota(**kw) + # Let DB generate ID if it isn't specified explicitly + if 'id' not in kw: + del quota['id'] + dbapi = db_api.get_instance() + + return dbapi.quota_create(context, quota) diff --git a/mogan/tests/unit/objects/test_objects.py b/mogan/tests/unit/objects/test_objects.py index 5eb1c6df..1678cb6d 100644 --- a/mogan/tests/unit/objects/test_objects.py +++ b/mogan/tests/unit/objects/test_objects.py @@ -389,7 +389,8 @@ expected_object_fingerprints = { 'MyObj': '1.1-aad62eedc5a5cc8bcaf2982c285e753f', 'FakeNode': '1.0-07813a70fee67557d8a71ad96f31cee7', 'InstanceNic': '1.0-78744332fe105f9c1796dc5295713d9f', - 'InstanceNics': '1.0-33a2e1bb91ad4082f9f63429b77c1244' + 'InstanceNics': '1.0-33a2e1bb91ad4082f9f63429b77c1244', + 'Quota': '1.0-c8caa082f4d726cb63fdc5943f7cd186', }