From 8f7d615e94e007382443515fbd202b47bf44d412 Mon Sep 17 00:00:00 2001 From: Mike Fedosin Date: Sat, 29 Jul 2017 16:01:08 +0300 Subject: [PATCH] Implement dynamic quotas This patch adds 3 new api calls PUT /quotas - set new quotas for specific projects GET /quotas - get info about all available quotas GET /quotas/{project_id} - get info about quotas for specific project Implements: blueprint glare-quotas Change-Id: Icd0c960bac897d48b8502dc39d77b6cffad0d6e4 --- glare/api/v1/resource.py | 115 ++++ glare/api/v1/router.py | 25 + glare/common/policy.py | 7 + .../versions/004_add_quota_tables.py | 48 ++ glare/db/sqlalchemy/api.py | 52 ++ glare/db/sqlalchemy/models.py | 16 +- glare/engine.py | 52 +- glare/notification.py | 5 +- glare/quota.py | 26 + glare/tests/functional/base.py | 2 +- glare/tests/functional/test_quotas.py | 646 ++++++++++++++++++ .../unit/db/migrations/test_migrations.py | 8 + glare/tests/unit/db/test_quota_functions.py | 78 ++- glare/tests/unit/test_quotas.py | 386 +++++++++++ 14 files changed, 1443 insertions(+), 23 deletions(-) create mode 100644 glare/db/migration/alembic_migrations/versions/004_add_quota_tables.py diff --git a/glare/api/v1/resource.py b/glare/api/v1/resource.py index cf7484f..d7826f2 100644 --- a/glare/api/v1/resource.py +++ b/glare/api/v1/resource.py @@ -17,6 +17,7 @@ deserialization of incoming requests.""" import json import jsonpatch +import jsonschema from oslo_config import cfg from oslo_log import log as logging from oslo_utils import encodeutils @@ -47,6 +48,39 @@ CONF.register_opts(list_configs) supported_versions = api_versioning.VersionedResource.supported_versions +QUOTA_SCHEMA = { + 'type': 'object', + 'properties': { + 'quota_name': { + u'maxLength': 255, + u'minLength': 1, + u'pattern': u'^[^:]*:?[^:]*$', # can have only 1 or 0 ':' + u'type': u'string'}, + 'quota_value': {'type': 'integer', u'minimum': -1}, + }, + 'required': ['quota_name', 'quota_value'] +} + +QUOTA_INPUT_SCHEMA = { + "$schema": "http://json-schema.org/draft-04/schema#", + "items": { + "properties": { + "project_id": { + u'maxLength': 255, + u'minLength': 1, + "type": "string" + }, + "project_quotas": { + "items": QUOTA_SCHEMA, + "type": "array" + } + }, + "type": "object", + "required": ["project_id", "project_quotas"] + }, + "type": "array" +} + class RequestDeserializer(api_versioning.VersionedResource, wsgi.JSONRequestDeserializer): @@ -203,6 +237,24 @@ class RequestDeserializer(api_versioning.VersionedResource, 'content_type': content_type, 'content_length': content_length} + @supported_versions(min_ver='1.1') + def set_quotas(self, req): + self._get_content_type(req, expected=['application/json']) + body = self._get_request_body(req) + try: + jsonschema.validate(body, QUOTA_INPUT_SCHEMA) + except jsonschema.exceptions.ValidationError as e: + raise exc.BadRequest(e) + values = {} + for item in body: + project_id = item['project_id'] + values[project_id] = {} + for quota in item['project_quotas']: + values[project_id][quota['quota_name']] = quota['quota_value'] + return {'values': values} + + # TODO(mfedosin) add pagination to list of quotas + def log_request_progress(f): def log_decorator(self, req, *args, **kwargs): @@ -401,6 +453,38 @@ class ArtifactsController(api_versioning.VersionedResource): return self.engine.delete_external_blob( req.context, type_name, artifact_id, field_name, blob_key) + @supported_versions(min_ver='1.1') + @log_request_progress + def set_quotas(self, req, values): + """Set quota records in Glare. + + :param req: user request + :param values: list with quota values to set + :return: definition of created quota + """ + return self.engine.set_quotas(req.context, values) + + @supported_versions(min_ver='1.1') + @log_request_progress + def list_all_quotas(self, req): + """Get detailed info about all available quotas. + + :param req: user request + :return: definition of requested quotas for the project + """ + return self.engine.list_all_quotas(req.context) + + @supported_versions(min_ver='1.1') + @log_request_progress + def list_project_quotas(self, req, project_id): + """Get detailed info about project quotas. + + :param req: user request + :param project_id: id of the project for which to show quotas + :return: definition of requested quotas for the project + """ + return self.engine.list_project_quotas(req.context, project_id) + class ResponseSerializer(api_versioning.VersionedResource, wsgi.JSONResponseSerializer): @@ -511,6 +595,37 @@ class ResponseSerializer(api_versioning.VersionedResource, def delete_external_blob(self, response, result): self._prepare_json_response(response, result) + @staticmethod + def _serialize_quota(quotas): + res = [] + for project_id, project_quotas in quotas.items(): + qouta_list = [] + for qouta_name, quota_value in project_quotas.items(): + qouta_list.append({ + 'quota_name': qouta_name, + 'quota_value': quota_value, + }) + res.append({ + 'project_id': project_id, + 'project_quotas': qouta_list + }) + return res + + @supported_versions(min_ver='1.1') + def set_quotas(self, response, quota): + quota = self._serialize_quota(quota) + self._prepare_json_response(response, quota) + + @supported_versions(min_ver='1.1') + def list_all_quotas(self, response, quota): + quota = self._serialize_quota(quota) + self._prepare_json_response(response, quota) + + @supported_versions(min_ver='1.1') + def list_project_quotas(self, response, quota): + quota = self._serialize_quota(quota) + self._prepare_json_response(response, quota) + def create_resource(): """Artifact resource factory method.""" diff --git a/glare/api/v1/router.py b/glare/api/v1/router.py index 121b3cd..84efb07 100644 --- a/glare/api/v1/router.py +++ b/glare/api/v1/router.py @@ -101,4 +101,29 @@ class API(wsgi.Router): action='reject', allowed_methods='GET, PUT, DELETE') + # ---quotas--- + mapper.connect('/quotas', + controller=glare_resource, + action='set_quotas', + conditions={'method': ['PUT']}) + mapper.connect('/quotas', + controller=glare_resource, + action='list_all_quotas', + conditions={'method': ['GET']}, + body_reject=True) + mapper.connect('/quotas', + controller=reject_method_resource, + action='reject', + allowed_methods='PUT, GET') + + mapper.connect('/quotas/{project_id}', + controller=glare_resource, + action='list_project_quotas', + conditions={'method': ['GET']}, + body_reject=True) + mapper.connect('/quotas/{project_id}', + controller=reject_method_resource, + action='reject', + allowed_methods='GET') + super(API, self).__init__(mapper) diff --git a/glare/common/policy.py b/glare/common/policy.py index e3ba51d..a4a5b71 100644 --- a/glare/common/policy.py +++ b/glare/common/policy.py @@ -83,6 +83,13 @@ artifact_policy_rules = [ policy.RuleDefault("artifact:delete_blob", "rule:admin_or_owner", "Policy to delete blob with external location " "from artifact"), + policy.RuleDefault("artifact:set_quotas", "rule:context_is_admin", + "Policy to set quotas for projects"), + policy.RuleDefault("artifact:list_all_quotas", "rule:context_is_admin", + "Policy to list all quotas for all projects"), + policy.RuleDefault("artifact:list_project_quotas", + "project_id:%(project_id)s or rule:context_is_admin", + "Policy to get info about project quotas"), ] diff --git a/glare/db/migration/alembic_migrations/versions/004_add_quota_tables.py b/glare/db/migration/alembic_migrations/versions/004_add_quota_tables.py new file mode 100644 index 0000000..2e896f1 --- /dev/null +++ b/glare/db/migration/alembic_migrations/versions/004_add_quota_tables.py @@ -0,0 +1,48 @@ +# Copyright 2017 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add quota tables + +Revision ID: 004 +Revises: 003 +Create Date: 2017-07-29 14:32:33.717353 + +""" + +# revision identifiers, used by Alembic. +revision = '004' +down_revision = '003' + +from alembic import op +import sqlalchemy as sa + +MYSQL_ENGINE = 'InnoDB' +MYSQL_CHARSET = 'utf8' + + +def upgrade(): + op.create_table( + 'glare_quotas', + sa.Column('project_id', sa.String(255), primary_key=True), + sa.Column('quota_name', sa.String(32), primary_key=True), + sa.Column('quota_value', sa.BigInteger(), nullable=False), + sa.PrimaryKeyConstraint('project_id', 'quota_name'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + +def downgrade(): + op.drop_table('glare_quotas') diff --git a/glare/db/sqlalchemy/api.py b/glare/db/sqlalchemy/api.py index 8df43fd..f46fa9d 100644 --- a/glare/db/sqlalchemy/api.py +++ b/glare/db/sqlalchemy/api.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import hashlib import operator import threading @@ -610,6 +611,57 @@ def calculate_uploaded_data(context, session, type_name=None): return query.order_by(None).scalar() or 0 +def _generate_quota_id(project_id, quota_name, type_name=None): + quota_id = b"%s:%s" % (project_id.encode(), quota_name.encode()) + if type_name is not None: + quota_id += b":%s" % type_name.encode() + return hashlib.md5(quota_id).hexdigest() + + +@retry(retry_on_exception=_retry_on_deadlock, wait_fixed=500, + stop_max_attempt_number=50) +@utils.no_4byte_params +def set_quotas(values, session): + """Create new quota instances in database""" + with session.begin(): + for project_id, project_quotas in values.items(): + + # reset all project quotas + session.query(models.ArtifactQuota).filter( + models.ArtifactQuota.project_id == project_id).delete() + + # generate new quotas + for quota_name, quota_value in project_quotas.items(): + q = models.ArtifactQuota() + q.project_id = project_id + q.quota_name = quota_name + q.quota_value = quota_value + session.add(q) + + # save all quotas + session.flush() + + return values + + +@retry(retry_on_exception=_retry_on_deadlock, wait_fixed=500, + stop_max_attempt_number=50) +def get_all_quotas(session, project_id=None): + """List all available quotas.""" + query = session.query(models.ArtifactQuota) + if project_id is not None: + query = query.filter( + models.ArtifactQuota.project_id == project_id) + quotas = query.order_by(models.ArtifactQuota.project_id).all() + + res = {} + for quota in quotas: + res.setdefault( + quota.project_id, {})[quota.quota_name] = quota.quota_value + + return res + + @retry(retry_on_exception=_retry_on_deadlock, wait_fixed=500, stop_max_attempt_number=50) @utils.no_4byte_params diff --git a/glare/db/sqlalchemy/models.py b/glare/db/sqlalchemy/models.py index 70280e0..cd6dbd0 100644 --- a/glare/db/sqlalchemy/models.py +++ b/glare/db/sqlalchemy/models.py @@ -255,17 +255,27 @@ class ArtifactBlobData(BASE, ArtifactBase): data = Column(LargeBinary(length=(2 ** 32) - 1), nullable=False) +class ArtifactQuota(BASE, ArtifactBase): + __tablename__ = 'glare_quotas' + __table_args__ = ( + {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'},) + project_id = Column(String(255), primary_key=True) + quota_name = Column(String(32), primary_key=True) + quota_value = Column(BigInteger().with_variant(Integer, "sqlite"), + nullable=False) + + def register_models(engine): """Create database tables for all models with the given engine.""" models = (Artifact, ArtifactTag, ArtifactProperty, ArtifactBlob, - ArtifactLock) + ArtifactLock, ArtifactQuota) for model in models: model.metadata.create_all(engine) def unregister_models(engine): """Drop database tables for all models with the given engine.""" - models = (ArtifactLock, ArtifactBlob, ArtifactProperty, ArtifactTag, - Artifact) + models = (ArtifactQuota, ArtifactLock, ArtifactBlob, ArtifactProperty, + ArtifactTag, Artifact) for model in models: model.metadata.drop_all(engine) diff --git a/glare/engine.py b/glare/engine.py index 0efdec0..10b3bef 100644 --- a/glare/engine.py +++ b/glare/engine.py @@ -60,12 +60,23 @@ class Engine(object): # register all artifact types registry.ArtifactRegistry.register_all_artifacts() - # generate all schemas + # generate all schemas and quotas self.schemas = {} + self.quotas = { + 'max_artifact_number': CONF.max_artifact_number, + 'max_uploaded_data': CONF.max_uploaded_data + } for name, type_list in registry.ArtifactRegistry.obj_classes().items(): type_name = type_list[0].get_type_name() self.schemas[type_name] = registry.ArtifactRegistry.\ get_artifact_type(type_name).gen_schemas() + type_conf_section = getattr(CONF, 'artifact_type:' + type_name) + if type_conf_section.max_artifact_number is not None: + self.quotas['max_artifact_number:' + type_name] = \ + type_conf_section.max_artifact_number + if type_conf_section.max_uploaded_data is not None: + self.quotas['max_uploaded_data:' + type_name] = \ + type_conf_section.max_uploaded_data lock_engine = locking.LockEngine(artifact_api.ArtifactLockApi()) @@ -673,3 +684,42 @@ class Engine(object): Notifier.notify(context, action_name, modified_af) return modified_af.to_dict() + + @staticmethod + def set_quotas(context, values): + """Set quota records in Glare. + + :param context: user request context + :param values: list with quota values to set + :return: definition of created quotas + """ + action_name = "artifact:set_quotas" + policy.authorize(action_name, {}, context) + qs = quota.set_quotas(values) + Notifier.notify(context, action_name, qs) + return qs + + def list_all_quotas(self, context): + """Get detailed info about all available quotas. + + :param context: user request context + :return: definition of requested quotas for the project + """ + action_name = "artifact:list_all_quotas" + policy.authorize(action_name, {}, context) + qs = quota.list_quotas() + qs[None] = self.quotas + return qs + + def list_project_quotas(self, context, project_id): + """Get detailed info about project quotas. + + :param context: user request context + :param project_id: id of the project for which to show quotas + :return: definition of requested quotas for the project + """ + action_name = "artifact:list_project_quotas" + policy.authorize(action_name, {'project_id': project_id}, context) + qs = self.quotas.copy() + qs.update(quota.list_quotas(project_id)[project_id]) + return {project_id: qs} diff --git a/glare/notification.py b/glare/notification.py index ccb2d08..1c883e2 100644 --- a/glare/notification.py +++ b/glare/notification.py @@ -59,8 +59,9 @@ class Notifier(object): """ af_notifier = cls._get_notifier() method = getattr(af_notifier, level.lower()) - method({}, "%s.%s" % (cls.SERVICE_NAME, event_type), - body.to_notification()) + if hasattr(body, 'to_notification'): + body = body.to_notification() + method({}, "%s.%s" % (cls.SERVICE_NAME, event_type), body) LOG.debug('Notification event %(event)s send successfully for ' 'request %(request)s', {'event': event_type, 'request': context.request_id}) diff --git a/glare/quota.py b/glare/quota.py index 4787979..bb70a29 100644 --- a/glare/quota.py +++ b/glare/quota.py @@ -31,6 +31,14 @@ def verify_artifact_count(context, type_name): type_limit = getattr( CONF, 'artifact_type:' + type_name).max_artifact_number + # update limits if they were reassigned for project + project_id = context.tenant + quotas = list_quotas(project_id).get(project_id, {}) + if 'max_artifact_number' in quotas: + global_limit = quotas['max_artifact_number'] + if 'max_artifact_number:' + type_name in quotas: + type_limit = quotas['max_artifact_number:' + type_name] + session = api.get_session() # the whole amount of created artifacts whole_number = api.count_artifact_number(context, session) @@ -71,6 +79,14 @@ def verify_uploaded_data_amount(context, type_name, data_amount=None): global_limit = CONF.max_uploaded_data type_limit = getattr(CONF, 'artifact_type:' + type_name).max_uploaded_data + # update limits if they were reassigned for project + project_id = context.tenant + quotas = list_quotas(project_id).get(project_id, {}) + if 'max_uploaded_data' in quotas: + global_limit = quotas['max_uploaded_data'] + if 'max_uploaded_data:' + type_name in quotas: + type_limit = quotas['max_uploaded_data:' + type_name] + session = api.get_session() # the whole amount of created artifacts whole_number = api.calculate_uploaded_data(context, session) @@ -107,3 +123,13 @@ def verify_uploaded_data_amount(context, type_name, data_amount=None): 'type_number': type_number} raise exception.RequestEntityTooLarge(msg) return res + + +def set_quotas(values): + session = api.get_session() + return api.set_quotas(values, session) + + +def list_quotas(project_id=None): + session = api.get_session() + return api.get_all_quotas(session, project_id) diff --git a/glare/tests/functional/base.py b/glare/tests/functional/base.py index 6d5ce0f..3a39168 100644 --- a/glare/tests/functional/base.py +++ b/glare/tests/functional/base.py @@ -74,7 +74,7 @@ class TestArtifact(functional.FunctionalTest): super(TestArtifact, self).tearDown() def _url(self, path): - if 'schemas' in path: + if path.startswith('/schemas') or path.startswith('/quotas'): return 'http://127.0.0.1:%d%s' % (self.glare_port, path) else: return 'http://127.0.0.1:%d/artifacts%s' % (self.glare_port, path) diff --git a/glare/tests/functional/test_quotas.py b/glare/tests/functional/test_quotas.py index b2eaebc..d929e79 100644 --- a/glare/tests/functional/test_quotas.py +++ b/glare/tests/functional/test_quotas.py @@ -15,6 +15,299 @@ from glare.tests.functional import base +class TestQuotasAPI(base.TestArtifact): + """Test quotas REST API.""" + + def setUp(self): + base.functional.FunctionalTest.setUp(self) + + self.glare_server.deployment_flavor = 'noauth' + + self.glare_server.max_uploaded_data = '10000' + self.glare_server.max_artifact_number = '150' + + self.glare_server.enabled_artifact_types = 'images,' \ + 'heat_templates,' \ + 'murano_packages,' \ + 'sample_artifact' + self.glare_server.custom_artifact_types_modules = ( + 'glare.tests.sample_artifact') + self.glare_server.artifact_type_section = """ +[artifact_type:sample_artifact] +max_uploaded_data = 3000 +[artifact_type:images] +max_uploaded_data = 15000 +max_artifact_number = 30 +[artifact_type:heat_templates] +max_artifact_number = 150 +[artifact_type:murano_packages] +max_uploaded_data = 10000 +max_artifact_number = 100 +""" + self.start_servers(**self.__dict__.copy()) + + def test_quota_api_wrong(self): + self.set_user('admin') + + url = '/quotas' + # try to set wrong values + values = [{"project1": "value1"}] + self.put(url=url, data=values, status=400) + + # no quota name + values = [ + { + "project_id": "project1", + "project_quotas": [ + { + "quota_value": 10 + } + ] + } + ] + self.put(url=url, data=values, status=400) + + # no quota value + values = [ + { + "project_id": "project1", + "project_quotas": [ + { + "quota_name": "max_artifact_number", + } + ] + } + ] + self.put(url=url, data=values, status=400) + + # no project id + values = [ + { + "project_quotas": [ + { + "quota_name": "max_artifact_number", + "quota_value": 10 + } + ] + } + ] + self.put(url=url, data=values, status=400) + + # no project quotas + values = [ + { + "project_id": "project1", + } + ] + self.put(url=url, data=values, status=400) + + # quota name has more than 1 : + values = [ + { + "project_id": "project1", + "project_quotas": [ + { + "quota_name": "max:artifact:number", + "quota_value": 10 + } + ] + } + ] + self.put(url=url, data=values, status=400) + + # too long quota name + values = [ + { + "project_id": "project1", + "project_quotas": [ + { + "quota_name": "a" * 256, + "quota_value": 10 + } + ] + } + ] + self.put(url=url, data=values, status=400) + + # too long project name + values = [ + { + "project_id": "a" * 256, + "project_quotas": [ + { + "quota_name": "max_artifact_number", + "quota_value": 10 + } + ] + } + ] + self.put(url=url, data=values, status=400) + + # negative quota value less than -1 + values = [ + { + "project_id": "project1", + "project_quotas": [ + { + "quota_name": "max_artifact_number", + "quota_value": -2 + } + ] + } + ] + self.put(url=url, data=values, status=400) + + # non-integer quota value + values = [ + { + "project_id": "project1", + "project_quotas": [ + { + "quota_name": "max_artifact_number", + "quota_value": "AAA" + } + ] + } + ] + self.put(url=url, data=values, status=400) + + values = [ + { + "project_id": "project1", + "project_quotas": [ + { + "quota_name": "max_artifact_number", + "quota_value": 10.5 + } + ] + } + ] + self.put(url=url, data=values, status=400) + + @staticmethod + def _deserialize_quotas(quotas): + values = {} + for item in quotas: + project_id = item['project_id'] + values[project_id] = {} + for quota in item['project_quotas']: + values[project_id][quota['quota_name']] = quota['quota_value'] + return values + + def test_quota_api(self): + self.set_user('admin') + user1_tenant_id = self.users['user1']['tenant_id'] + user2_tenant_id = self.users['user2']['tenant_id'] + admin_tenant_id = self.users['admin']['tenant_id'] + values = [ + { + "project_id": user1_tenant_id, + "project_quotas": [ + { + "quota_name": "max_artifact_number:images", + "quota_value": 3 + }, + { + "quota_name": "max_artifact_number:heat_templates", + "quota_value": 15 + }, + { + "quota_name": "max_artifact_number:murano_packages", + "quota_value": 10 + }, + { + "quota_name": "max_artifact_number", + "quota_value": 10 + } + ] + }, + { + "project_id": user2_tenant_id, + "project_quotas": [ + { + "quota_name": "max_artifact_number", + "quota_value": 10 + } + ] + }, + { + "project_id": admin_tenant_id, + "project_quotas": [ + { + "quota_name": "max_artifact_number", + "quota_value": 10 + } + ] + } + ] + + url = '/quotas' + # define several quotas + res = self._deserialize_quotas(self.put(url=url, data=values)) + self.assertEqual(self._deserialize_quotas(values), res) + + # get all quotas + res = self._deserialize_quotas(self.get(url=url)) + global_quotas = res.pop(None) + self.assertEqual({ + 'max_artifact_number': 150, + 'max_artifact_number:heat_templates': 150, + 'max_artifact_number:images': 30, + 'max_artifact_number:murano_packages': 100, + 'max_uploaded_data': 10000, + 'max_uploaded_data:images': 15000, + 'max_uploaded_data:murano_packages': 10000, + 'max_uploaded_data:sample_artifact': 3000}, global_quotas) + self.assertEqual(self._deserialize_quotas(values), res) + + # get user1 quotas + res = self._deserialize_quotas(self.get( + url='/quotas/' + user1_tenant_id)) + self.assertEqual({user1_tenant_id: { + 'max_artifact_number': 10, + 'max_artifact_number:heat_templates': 15, + 'max_artifact_number:images': 3, + 'max_artifact_number:murano_packages': 10, + 'max_uploaded_data': 10000, + 'max_uploaded_data:images': 15000, + 'max_uploaded_data:murano_packages': 10000, + 'max_uploaded_data:sample_artifact': 3000}}, res) + + # get admin quotas + res = self._deserialize_quotas(self.get( + url='/quotas/' + admin_tenant_id)) + self.assertEqual({admin_tenant_id: { + 'max_artifact_number': 10, + 'max_artifact_number:heat_templates': 150, + 'max_artifact_number:images': 30, + 'max_artifact_number:murano_packages': 100, + 'max_uploaded_data': 10000, + 'max_uploaded_data:images': 15000, + 'max_uploaded_data:murano_packages': 10000, + 'max_uploaded_data:sample_artifact': 3000}}, res) + + # user1 can't set quotas + self.set_user('user1') + self.put(url=url, data=values, status=403) + self.get(url=url, status=403) + + # user1 can get his quotas + res = self._deserialize_quotas(self.get( + url='/quotas/' + user1_tenant_id)) + self.assertEqual({user1_tenant_id: { + 'max_artifact_number': 10, + 'max_artifact_number:heat_templates': 15, + 'max_artifact_number:images': 3, + 'max_artifact_number:murano_packages': 10, + 'max_uploaded_data': 10000, + 'max_uploaded_data:images': 15000, + 'max_uploaded_data:murano_packages': 10000, + 'max_uploaded_data:sample_artifact': 3000}}, res) + + # user1 can't get user2 quotas + self.get(url=url + '/' + user2_tenant_id, status=403) + + class TestStaticQuotas(base.TestArtifact): """Test static quota limits.""" @@ -158,3 +451,356 @@ max_artifact_number = 10 self.put(url='/images/%s/image' % img1['id'], data=data, headers=headers) + + +class TestDynamicQuotas(base.TestArtifact): + """Test dynamic quota limits.""" + + def setUp(self): + base.functional.FunctionalTest.setUp(self) + + self.glare_server.deployment_flavor = 'noauth' + + self.glare_server.enabled_artifact_types = 'images,' \ + 'heat_templates,' \ + 'murano_packages,' \ + 'sample_artifact' + self.glare_server.custom_artifact_types_modules = ( + 'glare.tests.sample_artifact') + self.start_servers(**self.__dict__.copy()) + + def test_count_artifact_number(self): + self.set_user('admin') + user1_tenant_id = self.users['user1']['tenant_id'] + admin_tenant_id = self.users['admin']['tenant_id'] + values = [ + { + "project_id": user1_tenant_id, + "project_quotas": [ + { + "quota_name": "max_artifact_number:images", + "quota_value": 3 + }, + { + "quota_name": "max_artifact_number:heat_templates", + "quota_value": 15 + }, + { + "quota_name": "max_artifact_number:murano_packages", + "quota_value": 10 + }, + { + "quota_name": "max_artifact_number", + "quota_value": 10 + } + ] + }, + { + "project_id": admin_tenant_id, + "project_quotas": [ + { + "quota_name": "max_artifact_number", + "quota_value": 10 + } + ] + } + ] + url = '/quotas' + # define several quotas + self.put(url=url, data=values) + + self.set_user('user1') + # initially there are no artifacts + result = self.get('/all') + self.assertEqual([], result['all']) + + # create 3 images for user1 + for i in range(3): + img = self.create_artifact( + data={'name': 'img%d' % i}, type_name='images') + + # creation of another image fails because of artifact type limit + self.create_artifact( + data={'name': 'img4'}, type_name='images', status=403) + + # create 7 murano packages + for i in range(7): + self.create_artifact( + data={'name': 'mp%d' % i}, type_name='murano_packages') + + # creation of another package fails because of global limit + self.create_artifact( + data={'name': 'mp8'}, type_name='murano_packages', status=403) + + # delete an image and create another murano package work + self.delete('/images/%s' % img['id']) + self.create_artifact( + data={'name': 'mp8'}, type_name='murano_packages') + + # admin can create his own artifacts + self.set_user('admin') + for i in range(10): + self.create_artifact( + data={'name': 'ht%d' % i}, type_name='heat_templates') + + # creation of another heat template fails because of global limit + self.create_artifact( + data={'name': 'ht11'}, type_name='heat_templates', status=403) + + # disable global limit for user1 and try to create 15 heat templates + values = [ + { + "project_id": user1_tenant_id, + "project_quotas": [ + { + "quota_name": "max_artifact_number:images", + "quota_value": 3 + }, + { + "quota_name": "max_artifact_number:heat_templates", + "quota_value": 15 + }, + { + "quota_name": "max_artifact_number:murano_packages", + "quota_value": 10 + }, + { + "quota_name": "max_artifact_number", + "quota_value": -1 + } + ] + } + ] + url = '/quotas' + self.put(url=url, data=values) + + self.set_user("user1") + for i in range(15): + self.create_artifact( + data={'name': 'ht%d' % i}, type_name='heat_templates') + + # creation of another heat template fails because of type limit + self.create_artifact( + data={'name': 'ht16'}, type_name='heat_templates', status=403) + + self.set_user("admin") + # disable type limit for heat templates and create 1 heat templates + values = [ + { + "project_id": user1_tenant_id, + "project_quotas": [ + { + "quota_name": "max_artifact_number:images", + "quota_value": 3 + }, + { + "quota_name": "max_artifact_number:heat_templates", + "quota_value": -1 + }, + { + "quota_name": "max_artifact_number:murano_packages", + "quota_value": 10 + }, + { + "quota_name": "max_artifact_number", + "quota_value": -1 + } + ] + } + ] + url = '/quotas' + self.put(url=url, data=values) + + # now user1 can create another heat template + self.set_user("user1") + self.create_artifact( + data={'name': 'ht16'}, type_name='heat_templates') + + def test_calculate_uploaded_data(self): + self.set_user('admin') + user1_tenant_id = self.users['user1']['tenant_id'] + admin_tenant_id = self.users['admin']['tenant_id'] + values = [ + { + "project_id": user1_tenant_id, + "project_quotas": [ + { + "quota_name": "max_uploaded_data:images", + "quota_value": 1500 + }, + { + "quota_name": "max_uploaded_data:sample_artifact", + "quota_value": 300 + }, + { + "quota_name": "max_uploaded_data:murano_packages", + "quota_value": 1000 + }, + { + "quota_name": "max_uploaded_data", + "quota_value": 1000 + } + ] + }, + { + "project_id": admin_tenant_id, + "project_quotas": [ + { + "quota_name": "max_uploaded_data", + "quota_value": 1000 + } + ] + } + ] + url = '/quotas' + # define several quotas + self.put(url=url, data=values) + + headers = {'Content-Type': 'application/octet-stream'} + + self.set_user('user1') + # initially there are no artifacts + result = self.get('/all') + self.assertEqual([], result['all']) + + # create 2 sample artifacts for user1 + art1 = self.create_artifact(data={'name': 'art1'}) + art2 = self.create_artifact(data={'name': 'art2'}) + + # create 3 images for user1 + img1 = self.create_artifact(data={'name': 'img1'}, type_name='images') + img2 = self.create_artifact(data={'name': 'img2'}, type_name='images') + img3 = self.create_artifact(data={'name': 'img3'}, type_name='images') + + # upload to art1 fails now because of type limit + data = 'a' * 301 + self.put(url='/sample_artifact/%s/blob' % art1['id'], + data=data, + status=413, + headers=headers) + + # upload to img1 fails now because of global limit + data = 'a' * 1001 + self.put(url='/images/%s/image' % img1['id'], + data=data, + status=413, + headers=headers) + + # upload 300 bytes to 'blob' of art1 + data = 'a' * 300 + self.put(url='/sample_artifact/%s/blob' % art1['id'], + data=data, + headers=headers) + + # upload another blob to art1 fails because of type limit + self.put(url='/sample_artifact/%s/dict_of_blobs/blob' % art1['id'], + data='a', + status=413, + headers=headers) + + # upload to art2 fails now because of type limit + self.put(url='/sample_artifact/%s/dict_of_blobs/blob' % art2['id'], + data='a', + status=413, + headers=headers) + + # delete art1 and check that upload to art2 works + data = 'a' * 300 + self.delete('/sample_artifact/%s' % art1['id']) + self.put(url='/sample_artifact/%s/dict_of_blobs/blob' % art2['id'], + data=data, + headers=headers) + + # upload 700 bytes to img1 works + data = 'a' * 700 + self.put(url='/images/%s/image' % img1['id'], + data=data, + headers=headers) + + # upload to img2 fails because of global limit + self.put(url='/images/%s/image' % img2['id'], + data='a', + status=413, + headers=headers) + + # admin can upload data to his images + self.set_user('admin') + img1 = self.create_artifact(data={'name': 'img1'}, type_name='images') + data = 'a' * 1000 + self.put(url='/images/%s/image' % img1['id'], + data=data, + headers=headers) + + # disable global limit and try upload data from user1 again + values = [ + { + "project_id": user1_tenant_id, + "project_quotas": [ + { + "quota_name": "max_uploaded_data:images", + "quota_value": 1500 + }, + { + "quota_name": "max_uploaded_data:sample_artifact", + "quota_value": 300 + }, + { + "quota_name": "max_uploaded_data:murano_packages", + "quota_value": 1000 + }, + { + "quota_name": "max_uploaded_data", + "quota_value": -1 + } + ] + } + ] + url = '/quotas' + self.put(url=url, data=values) + + self.set_user("user1") + data = 'a' * 800 + self.put(url='/images/%s/image' % img2['id'], + data=data, + headers=headers) + + # uploading more fails because of image type limit + data = 'a' + self.put(url='/images/%s/image' % img3['id'], + data=data, + headers=headers, + status=413) + + # disable type limit and try upload data from user1 again + self.set_user("admin") + values = [ + { + "project_id": user1_tenant_id, + "project_quotas": [ + { + "quota_name": "max_uploaded_data:images", + "quota_value": -1 + }, + { + "quota_name": "max_uploaded_data:sample_artifact", + "quota_value": 300 + }, + { + "quota_name": "max_uploaded_data:murano_packages", + "quota_value": 1000 + }, + { + "quota_name": "max_uploaded_data", + "quota_value": -1 + } + ] + } + ] + url = '/quotas' + self.put(url=url, data=values) + + self.set_user("user1") + data = 'a' * 1000 + self.put(url='/images/%s/image' % img3['id'], + data=data, + headers=headers) diff --git a/glare/tests/unit/db/migrations/test_migrations.py b/glare/tests/unit/db/migrations/test_migrations.py index b5e9b37..977910c 100644 --- a/glare/tests/unit/db/migrations/test_migrations.py +++ b/glare/tests/unit/db/migrations/test_migrations.py @@ -239,6 +239,14 @@ class GlareMigrationsCheckers(object): self.assert_table(engine, 'glare_blob_data', locks_indices, locks_columns) + def _check_004(self, engine, data): + quota_indices = [] + quota_columns = ['project_id', + 'quota_name', + 'quota_value'] + self.assert_table(engine, 'glare_quotas', quota_indices, + quota_columns) + class TestMigrationsMySQL(GlareMigrationsCheckers, WalkVersionsMixin, diff --git a/glare/tests/unit/db/test_quota_functions.py b/glare/tests/unit/db/test_quota_functions.py index 81d1ebe..dd30810 100644 --- a/glare/tests/unit/db/test_quota_functions.py +++ b/glare/tests/unit/db/test_quota_functions.py @@ -21,12 +21,14 @@ from glare.tests.unit import base class TestQuotaFunctions(base.BaseTestArtifactAPI): """Test quota db functions.""" - def test_count_artifact_number(self): - session = api.get_session() + def setUp(self): + super(TestQuotaFunctions, self).setUp() + self.session = api.get_session() + def test_count_artifact_number(self): # initially there are no artifacts self.assertEqual(0, api.count_artifact_number( - self.req.context, session)) + self.req.context, self.session)) # create 5 images, 3 heat templates, 2 murano packages and 7 samples amount = { @@ -48,19 +50,17 @@ class TestQuotaFunctions(base.BaseTestArtifactAPI): # count numbers for each type for type_name in amount: num = api.count_artifact_number( - self.req.context, session, type_name) + self.req.context, self.session, type_name) self.assertEqual(amount[type_name], num) # count the whole amount of artifacts self.assertEqual(17, api.count_artifact_number( - self.req.context, session)) + self.req.context, self.session)) def test_calculate_uploaded_data(self): - session = api.get_session() - # initially there is no data self.assertEqual(0, api.calculate_uploaded_data( - self.req.context, session)) + self.req.context, self.session)) # create a sample artifact art1 = self.controller.create( @@ -71,7 +71,7 @@ class TestQuotaFunctions(base.BaseTestArtifactAPI): self.req, 'sample_artifact', art1['id'], 'blob', BytesIO(b'a' * 10), 'application/octet-stream') self.assertEqual(10, api.calculate_uploaded_data( - self.req.context, session)) + self.req.context, self.session)) # upload 3 blobs to dict_of_blobs with 25, 35 and 45 bytes respectively self.controller.upload_blob( @@ -84,7 +84,7 @@ class TestQuotaFunctions(base.BaseTestArtifactAPI): self.req, 'sample_artifact', art1['id'], 'dict_of_blobs/blob3', BytesIO(b'a' * 45), 'application/octet-stream') self.assertEqual(115, api.calculate_uploaded_data( - self.req.context, session)) + self.req.context, self.session)) # create another sample artifact and upload 100 bytes there art2 = self.controller.create( @@ -93,7 +93,7 @@ class TestQuotaFunctions(base.BaseTestArtifactAPI): self.req, 'sample_artifact', art2['id'], 'blob', BytesIO(b'a' * 100), 'application/octet-stream') self.assertEqual(215, api.calculate_uploaded_data( - self.req.context, session)) + self.req.context, self.session)) # create image and upload 150 bytes there img1 = self.controller.create( @@ -103,13 +103,13 @@ class TestQuotaFunctions(base.BaseTestArtifactAPI): BytesIO(b'a' * 150), 'application/octet-stream') # the whole amount of uploaded data is 365 bytes self.assertEqual(365, api.calculate_uploaded_data( - self.req.context, session)) + self.req.context, self.session)) # 215 bytes for sample_artifact self.assertEqual(215, api.calculate_uploaded_data( - self.req.context, session, 'sample_artifact')) + self.req.context, self.session, 'sample_artifact')) # 150 bytes for images self.assertEqual(150, api.calculate_uploaded_data( - self.req.context, session, 'images')) + self.req.context, self.session, 'images')) # create an artifact from another user and check that it's not included # for the original user @@ -123,7 +123,53 @@ class TestQuotaFunctions(base.BaseTestArtifactAPI): # original user still has 365 bytes self.assertEqual(365, api.calculate_uploaded_data( - self.req.context, session)) + self.req.context, self.session)) # user2 has 1000 self.assertEqual( - 1000, api.calculate_uploaded_data(req.context, session)) + 1000, api.calculate_uploaded_data(req.context, self.session)) + + def test_quota_operations(self): + # create several quotas + values = { + "project1": { + "max_uploaded_data": 1000, + "max_uploaded_data:images": 500, + "max_artifact_number": 10 + }, + "project2": { + "max_uploaded_data": 1000, + "max_uploaded_data:sample_artifact": 500, + "max_artifact_number": 20 + }, + "project3": { + "max_uploaded_data": 1000 + } + } + + res = api.set_quotas(values, self.session) + self.assertEqual(values, res) + + res = api.get_all_quotas(self.session) + self.assertEqual(values, res) + + # Redefine quotas + new_values = { + "project1": { + "max_uploaded_data": 200, + "max_uploaded_data:images": 1000, + "max_artifact_number": 30, + "max_artifact_number:images": 20 + }, + "project2": {}, + } + + res = api.set_quotas(new_values, self.session) + self.assertEqual(new_values, res) + + # project3 should remain unchanged + new_values['project3'] = {"max_uploaded_data": 1000} + # project 2 quotas removed + new_values.pop('project2') + + res = api.get_all_quotas(self.session) + self.assertEqual(new_values, res) diff --git a/glare/tests/unit/test_quotas.py b/glare/tests/unit/test_quotas.py index c8c1e52..97f2fbd 100644 --- a/glare/tests/unit/test_quotas.py +++ b/glare/tests/unit/test_quotas.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import mock from six import BytesIO from glare.common import exception +from glare.common import store_api from glare.tests.unit import base @@ -192,3 +194,387 @@ class TestStaticQuotas(base.BaseTestArtifactAPI): self.controller.upload_blob( user1_req, 'images', img3['id'], 'image', BytesIO(b'a' * 1000), 'application/octet-stream', 1000) + + +class TestDynamicQuotas(base.BaseTestArtifactAPI): + """Test dynamic quota limits.""" + + def test_count_artifact_number(self): + user1_req = self.get_fake_request(self.users['user1']) + user2_req = self.get_fake_request(self.users['user2']) + # initially there are no artifacts + self.assertEqual( + 0, len(self.controller.list(user1_req, 'all')['artifacts'])) + self.assertEqual( + 0, len(self.controller.list(user2_req, 'all')['artifacts'])) + + values = { + user1_req.context.tenant: { + "max_artifact_number:images": 3, + "max_artifact_number:heat_templates": 15, + "max_artifact_number:murano_packages": 10, + "max_artifact_number": 10 + }, + user2_req.context.tenant: { + "max_artifact_number": 10 + } + } + + admin_req = self.get_fake_request(self.users["admin"]) + # define several quotas + self.controller.set_quotas(admin_req, values) + + # create 3 images for user1 + for i in range(3): + img = self.controller.create( + user1_req, 'images', {'name': 'img%d' % i}) + + # creation of another image fails because of artifact type limit + self.assertRaises(exception.Forbidden, self.controller.create, + user1_req, 'images', {'name': 'img4'}) + + # create 7 murano packages + for i in range(7): + self.controller.create( + user1_req, 'murano_packages', {'name': 'mp%d' % i}) + + # creation of another package fails because of global limit + self.assertRaises(exception.Forbidden, self.controller.create, + user1_req, 'murano_packages', {'name': 'mp8'}) + + # delete an image and create another murano package work + self.controller.delete(user1_req, 'images', img['id']) + self.controller.create(user1_req, 'murano_packages', {'name': 'mp8'}) + + # user2 can create his own artifacts + for i in range(10): + self.controller.create( + user2_req, 'heat_templates', {'name': 'ht%d' % i}) + + # creation of another heat template fails because of global limit + self.assertRaises(exception.Forbidden, self.controller.create, + user2_req, 'heat_templates', {'name': 'ht11'}) + + # disable global limit for user1 and try to create 15 heat templates + values = { + user1_req.context.tenant: { + "max_artifact_number:images": 3, + "max_artifact_number:heat_templates": 15, + "max_artifact_number:murano_packages": 10, + "max_artifact_number": -1 + } + } + self.controller.set_quotas(admin_req, values) + + for i in range(15): + self.controller.create( + user1_req, 'heat_templates', {'name': 'ht%d' % i}) + + # creation of another heat template fails because of type limit + self.assertRaises(exception.Forbidden, self.controller.create, + user1_req, 'heat_templates', {'name': 'ht16'}) + + # disable type limit for heat templates and create 1 heat templates + values = { + user1_req.context.tenant: { + "max_artifact_number:images": 3, + "max_artifact_number:heat_templates": -1, + "max_artifact_number:murano_packages": 10, + "max_artifact_number": -1 + } + } + self.controller.set_quotas(admin_req, values) + + # now user1 can create another heat template + self.controller.create( + user1_req, 'heat_templates', {'name': 'ht16'}) + + def test_calculate_uploaded_data(self): + user1_req = self.get_fake_request(self.users['user1']) + user2_req = self.get_fake_request(self.users['user2']) + # initially there are no artifacts + self.assertEqual( + 0, len(self.controller.list(user1_req, 'all')['artifacts'])) + self.assertEqual( + 0, len(self.controller.list(user2_req, 'all')['artifacts'])) + + values = { + user1_req.context.tenant: { + "max_uploaded_data:images": 1500, + "max_uploaded_data:sample_artifact": 300, + "max_uploaded_data:murano_packages": 1000, + "max_uploaded_data": 1000 + }, + user2_req.context.tenant: { + "max_uploaded_data": 1000 + } + } + + admin_req = self.get_fake_request(self.users["admin"]) + # define several quotas + self.controller.set_quotas(admin_req, values) + # create 2 sample artifacts for user 1 + art1 = self.controller.create( + user1_req, 'sample_artifact', {'name': 'art1'}) + art2 = self.controller.create( + user1_req, 'sample_artifact', {'name': 'art2'}) + + # create 3 images for user1 + img1 = self.controller.create( + user1_req, 'images', {'name': 'img1'}) + img2 = self.controller.create( + user1_req, 'images', {'name': 'img2'}) + img3 = self.controller.create( + user1_req, 'images', {'name': 'img3'}) + + # upload to art1 fails now because of type limit + self.assertRaises( + exception.RequestEntityTooLarge, self.controller.upload_blob, + user1_req, 'sample_artifact', art1['id'], 'blob', + BytesIO(b'a' * 301), 'application/octet-stream', 301) + + # upload to img1 fails now because of global limit + self.assertRaises( + exception.RequestEntityTooLarge, self.controller.upload_blob, + user1_req, 'images', img1['id'], 'image', + BytesIO(b'a' * 1001), 'application/octet-stream', 1001) + + # upload 300 bytes to 'blob' of art1 + self.controller.upload_blob( + user1_req, 'sample_artifact', art1['id'], 'blob', + BytesIO(b'a' * 300), 'application/octet-stream', + content_length=300) + + # upload another blob to art1 fails because of type limit + self.assertRaises( + exception.RequestEntityTooLarge, self.controller.upload_blob, + user1_req, 'sample_artifact', art1['id'], + 'dict_of_blobs/blob', BytesIO(b'a'), + 'application/octet-stream', 1) + + # upload to art2 fails now because of type limit + self.assertRaises( + exception.RequestEntityTooLarge, self.controller.upload_blob, + user1_req, 'sample_artifact', art2['id'], 'blob', + BytesIO(b'a'), 'application/octet-stream', 1) + + # delete art1 and check that upload to art2 works + self.controller.delete(user1_req, 'sample_artifact', art1['id']) + self.controller.upload_blob( + user1_req, 'sample_artifact', art2['id'], 'blob', + BytesIO(b'a' * 300), 'application/octet-stream', 300) + + # upload 700 bytes to img1 works + self.controller.upload_blob( + user1_req, 'images', img1['id'], 'image', + BytesIO(b'a' * 700), 'application/octet-stream', 700) + + # upload to img2 fails because of global limit + self.assertRaises( + exception.RequestEntityTooLarge, self.controller.upload_blob, + user1_req, 'images', img2['id'], 'image', + BytesIO(b'a'), 'application/octet-stream', 1) + + # user2 can upload data to images + img1 = self.controller.create( + user2_req, 'images', {'name': 'img1'}) + self.controller.upload_blob( + user2_req, 'images', img1['id'], 'image', + BytesIO(b'a' * 1000), 'application/octet-stream', 1000) + + # disable global limit and try upload data from user1 again + values = { + user1_req.context.tenant: { + "max_uploaded_data:images": 1500, + "max_uploaded_data:sample_artifact": 300, + "max_uploaded_data:murano_packages": 1000, + "max_uploaded_data": -1 + } + } + self.controller.set_quotas(admin_req, values) + + self.controller.upload_blob( + user1_req, 'images', img2['id'], 'image', + BytesIO(b'a' * 800), 'application/octet-stream', 800) + + # uploading more fails because of image type limit + self.assertRaises( + exception.RequestEntityTooLarge, self.controller.upload_blob, + user1_req, 'images', img3['id'], 'image', + BytesIO(b'a'), 'application/octet-stream', 1) + + # disable type limit and try upload data from user1 again + values = { + user1_req.context.tenant: { + "max_uploaded_data:images": -1, + "max_uploaded_data:sample_artifact": 300, + "max_uploaded_data:murano_packages": 1000, + "max_uploaded_data": -1 + } + } + self.controller.set_quotas(admin_req, values) + self.controller.upload_blob( + user1_req, 'images', img3['id'], 'image', + BytesIO(b'a' * 1000), 'application/octet-stream', 1000) + + def test_quota_upload_no_content_length(self): + user1_req = self.get_fake_request(self.users['user1']) + user2_req = self.get_fake_request(self.users['user2']) + admin_req = self.get_fake_request(self.users['admin']) + + values = { + user1_req.context.tenant: { + "max_uploaded_data:sample_artifact": 20, + "max_uploaded_data": 5 + }, + user2_req.context.tenant: { + "max_uploaded_data:sample_artifact": 7, + "max_uploaded_data": -1 + }, + admin_req.context.tenant: { + "max_uploaded_data:sample_artifact": -1, + "max_uploaded_data": -1 + } + } + + # define several quotas + self.controller.set_quotas(admin_req, values) + + # create a sample artifacts for user 1 + art1 = self.controller.create( + user1_req, 'sample_artifact', {'name': 'art1'}) + + # Max small_blob size is 10. User1 global quota is 5. + # Since user doesn't specify how many bytes he wants to upload, + # engine can't verify it before upload. Therefore it allocates + # 5 available bytes for user and begins upload. If uploaded data + # amount exceeds this limit RequestEntityTooLarge is raised and + # upload fails. + with mock.patch( + 'glare.common.store_api.save_blob_to_store', + side_effect=store_api.save_blob_to_store) as mocked_save: + data = BytesIO(b'a' * 10) + self.assertRaises( + exception.RequestEntityTooLarge, + self.controller.upload_blob, + user1_req, 'sample_artifact', art1['id'], 'small_blob', + data, 'application/octet-stream', + content_length=None) + mocked_save.assert_called_once_with( + mock.ANY, data, user1_req.context, 5, store_type='database') + + # check that blob wasn't uploaded + self.assertIsNone( + self.controller.show( + user1_req, 'sample_artifact', art1['id'])['small_blob']) + + # try to upload with smaller amount that doesn't exceeds quota + with mock.patch( + 'glare.common.store_api.save_blob_to_store', + side_effect=store_api.save_blob_to_store) as mocked_save: + data = BytesIO(b'a' * 4) + self.controller.upload_blob( + user1_req, 'sample_artifact', art1['id'], 'small_blob', + data, 'application/octet-stream', + content_length=None) + mocked_save.assert_called_once_with( + mock.ANY, data, user1_req.context, 5, store_type='database') + + # check that blob was uploaded + blob = self.controller.show( + user1_req, 'sample_artifact', art1['id'])['small_blob'] + self.assertEqual(4, blob['size']) + self.assertEqual('active', blob['status']) + + # create a sample artifacts for user 2 + art2 = self.controller.create( + user2_req, 'sample_artifact', {'name': 'art2'}) + + # Max small_blob size is 10. User1 has no global quota, but his + # type quota is 7. + # Since user doesn't specify how many bytes he wants to upload, + # engine can't verify it before upload. Therefore it allocates + # 7 available bytes for user and begins upload. If uploaded data + # amount exceeds this limit RequestEntityTooLarge is raised and + # upload fails. + with mock.patch( + 'glare.common.store_api.save_blob_to_store', + side_effect=store_api.save_blob_to_store) as mocked_save: + data = BytesIO(b'a' * 10) + self.assertRaises( + exception.RequestEntityTooLarge, + self.controller.upload_blob, + user2_req, 'sample_artifact', art2['id'], 'small_blob', + data, 'application/octet-stream', + content_length=None) + mocked_save.assert_called_once_with( + mock.ANY, data, user2_req.context, 7, store_type='database') + + # check that blob wasn't uploaded + self.assertIsNone( + self.controller.show( + user2_req, 'sample_artifact', art2['id'])['small_blob']) + + # try to upload with smaller amount that doesn't exceeds quota + with mock.patch( + 'glare.common.store_api.save_blob_to_store', + side_effect=store_api.save_blob_to_store) as mocked_save: + data = BytesIO(b'a' * 7) + self.controller.upload_blob( + user2_req, 'sample_artifact', art2['id'], 'small_blob', + data, 'application/octet-stream', + content_length=None) + mocked_save.assert_called_once_with( + mock.ANY, data, user2_req.context, 7, store_type='database') + + # check that blob was uploaded + blob = self.controller.show( + user2_req, 'sample_artifact', art2['id'])['small_blob'] + self.assertEqual(7, blob['size']) + self.assertEqual('active', blob['status']) + + # create a sample artifacts for admin + arta = self.controller.create( + user2_req, 'sample_artifact', {'name': 'arta'}) + + # Max small_blob size is 10. Admin has no quotas at all. + # Since admin doesn't specify how many bytes he wants to upload, + # engine can't verify it before upload. Therefore it allocates + # 10 available bytes (max allowed small_blob size) for him and begins + # upload. If uploaded data amount exceeds this limit + # RequestEntityTooLarge is raised and upload fails. + with mock.patch( + 'glare.common.store_api.save_blob_to_store', + side_effect=store_api.save_blob_to_store) as mocked_save: + data = BytesIO(b'a' * 11) + self.assertRaises( + exception.RequestEntityTooLarge, + self.controller.upload_blob, + admin_req, 'sample_artifact', arta['id'], 'small_blob', + data, 'application/octet-stream', + content_length=None) + mocked_save.assert_called_once_with( + mock.ANY, data, admin_req.context, 10, store_type='database') + + # check that blob wasn't uploaded + self.assertIsNone( + self.controller.show( + admin_req, 'sample_artifact', arta['id'])['small_blob']) + + # try to upload with smaller amount that doesn't exceeds quota + with mock.patch( + 'glare.common.store_api.save_blob_to_store', + side_effect=store_api.save_blob_to_store) as mocked_save: + data = BytesIO(b'a' * 10) + self.controller.upload_blob( + admin_req, 'sample_artifact', arta['id'], 'small_blob', + data, 'application/octet-stream', + content_length=None) + mocked_save.assert_called_once_with( + mock.ANY, data, admin_req.context, 10, store_type='database') + + # check that blob was uploaded + blob = self.controller.show( + admin_req, 'sample_artifact', arta['id'])['small_blob'] + self.assertEqual(10, blob['size']) + self.assertEqual('active', blob['status'])