Add a storage quota
This patch adds a storage quota that is applied against the sum total of a users storage consumption against all configured storage systems. A single quota is applied to all users via the configuration option 'total_storage_quota'. Most of the patch is about enforcement so when a separate service for quota management emerges in OpenStack the per user value to enforce can be obtained from that service but the enforcement code will remain the same. blueprint glance-basic-quotas docImpact Change-Id: I251832f7372c70942be6f0c6aa12285145dd7c18
This commit is contained in:
John Bresnahan
parent
1ed4feca1d
commit
e3e4f4d927
@ -331,6 +331,19 @@ Maximum image size, in bytes, which can be uploaded through the Glance API serve
|
||||
**IMPORTANT NOTE**: this value should only be increased after careful consideration
|
||||
and must be set to a value under 8 EB (9223372036854775808).
|
||||
|
||||
Configuring Glance User Storage Quota
|
||||
-------------------------------------
|
||||
|
||||
The following configuration option is specified in the
|
||||
``glance-api.conf`` config file in the section ``[DEFAULT]``.
|
||||
|
||||
* ``user_storage_quota``
|
||||
|
||||
Optional. Default: 0 (Unlimited).
|
||||
|
||||
This value specifies the maximum amount of bytes that each user can use
|
||||
across all storage systems.
|
||||
|
||||
Configuring the Filesystem Storage Backend
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
||||
@ -104,6 +104,11 @@ workers = 1
|
||||
#disk_formats=ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso
|
||||
|
||||
|
||||
# Set a system wide quota for every user. This value is the total number
|
||||
# of bytes that a user can use across all storage systems. A value of
|
||||
# 0 means unlimited.
|
||||
#user_storage_quota = 0
|
||||
|
||||
# ================= Syslog Options ============================
|
||||
|
||||
# Send logs to syslog (/dev/log) instead of to file specified
|
||||
|
||||
@ -13,10 +13,13 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
from glance.common import exception
|
||||
from glance.openstack.common import log as logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
def size_checked_iter(response, image_meta, expected_size, image_iter,
|
||||
@ -76,3 +79,59 @@ def image_send_notification(bytes_written, expected_size, image_meta, request,
|
||||
msg = _("An error occurred during image.send"
|
||||
" notification: %(err)s") % locals()
|
||||
LOG.error(msg)
|
||||
|
||||
|
||||
def get_remaining_quota(context, db_api, image_id=None):
|
||||
"""
|
||||
This method is called to see if the user is allowed to store an image
|
||||
of the given size in glance based on their quota and current usage.
|
||||
:param context:
|
||||
:param db_api: The db_api in use for this configuration
|
||||
:param image_id: The image that will be replaced with this new data size
|
||||
:return: The number of bytes the user has remaining under their quota.
|
||||
None means infinity
|
||||
"""
|
||||
|
||||
#NOTE(jbresnah) in the future this value will come from a call to
|
||||
# keystone.
|
||||
users_quota = CONF.user_storage_quota
|
||||
if users_quota <= 0:
|
||||
return None
|
||||
|
||||
usage = db_api.user_get_storage_usage(context,
|
||||
context.owner,
|
||||
image_id=image_id)
|
||||
return users_quota - usage
|
||||
|
||||
|
||||
def check_quota(context, image_size, db_api, image_id=None):
|
||||
"""
|
||||
This method is called to see if the user is allowed to store an image
|
||||
of the given size in glance based on their quota and current usage.
|
||||
:param context:
|
||||
:param image_size: The size of the image we hope to store
|
||||
:param db_api: The db_api in use for this configuration
|
||||
:param image_id: The image that will be replaced with this new data size
|
||||
:return:
|
||||
"""
|
||||
|
||||
remaining = get_remaining_quota(context, db_api, image_id=image_id)
|
||||
|
||||
if remaining is None:
|
||||
return
|
||||
|
||||
if image_size is None:
|
||||
#NOTE(jbresnah) When the image size is None it means that it is
|
||||
# not known. In this case the only time we will raise an
|
||||
# exception is when there is no room left at all, thus we know
|
||||
# it will not fit
|
||||
if remaining <= 0:
|
||||
raise exception.StorageQuotaFull(image_size=image_size,
|
||||
remaining=remaining)
|
||||
return
|
||||
|
||||
if image_size > remaining:
|
||||
raise exception.StorageQuotaFull(image_size=image_size,
|
||||
remaining=remaining)
|
||||
|
||||
return remaining
|
||||
|
||||
@ -21,6 +21,7 @@ import webob.exc
|
||||
from glance.common import exception
|
||||
from glance.openstack.common import excutils
|
||||
from glance.common import utils
|
||||
import glance.db
|
||||
import glance.openstack.common.log as logging
|
||||
import glance.registry.client.v1.api as registry
|
||||
import glance.store
|
||||
@ -79,7 +80,16 @@ def upload_data_to_store(req, image_meta, image_data, store, notifier):
|
||||
Upload image data to the store and cleans up on error.
|
||||
"""
|
||||
image_id = image_meta['id']
|
||||
|
||||
db_api = glance.db.get_api()
|
||||
image_size = image_meta.get('size', None)
|
||||
|
||||
try:
|
||||
remaining = glance.api.common.check_quota(
|
||||
req.context, image_size, db_api, image_id=image_id)
|
||||
if remaining is not None:
|
||||
image_data = utils.LimitingReader(image_data, remaining)
|
||||
|
||||
(location,
|
||||
size,
|
||||
checksum,
|
||||
@ -89,6 +99,18 @@ def upload_data_to_store(req, image_meta, image_data, store, notifier):
|
||||
image_meta['size'],
|
||||
store)
|
||||
|
||||
try:
|
||||
# recheck the quota in case there were simultaneous uploads that
|
||||
# did not provide the size
|
||||
glance.api.common.check_quota(
|
||||
req.context, size, db_api, image_id=image_id)
|
||||
except exception.StorageQuotaFull:
|
||||
LOG.info(_('Cleaning up %s after exceeding the quota %s')
|
||||
% image_id)
|
||||
glance.store.safe_delete_from_backend(
|
||||
location, req.context, image_meta['id'])
|
||||
raise
|
||||
|
||||
def _kill_mismatched(image_meta, attr, actual):
|
||||
supplied = image_meta.get(attr)
|
||||
if supplied and supplied != actual:
|
||||
@ -182,6 +204,16 @@ def upload_data_to_store(req, image_meta, image_data, store, notifier):
|
||||
request=req,
|
||||
content_type='text/plain')
|
||||
|
||||
except exception.StorageQuotaFull as e:
|
||||
msg = (_("Denying attempt to upload image because it exceeds the ."
|
||||
"quota: %s") % e)
|
||||
LOG.info(msg)
|
||||
safe_kill(req, image_id)
|
||||
notifier.error('image.upload', msg)
|
||||
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
|
||||
request=req,
|
||||
content_type='text/plain')
|
||||
|
||||
except webob.exc.HTTPError:
|
||||
#NOTE(bcwaldon): Ideally, we would just call 'raise' here,
|
||||
# but something in the above function calls is affecting the
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
|
||||
import webob.exc
|
||||
|
||||
import glance.api.common
|
||||
import glance.api.policy
|
||||
from glance.common import exception
|
||||
from glance.common import utils
|
||||
@ -76,6 +77,24 @@ class ImageDataController(object):
|
||||
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
|
||||
request=req)
|
||||
|
||||
except exception.StorageFull as e:
|
||||
msg = _("Image storage media is full: %s") % e
|
||||
LOG.error(msg)
|
||||
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
|
||||
request=req)
|
||||
|
||||
except exception.StorageQuotaFull as e:
|
||||
msg = _("Image exceeds the storage quota: %s") % e
|
||||
LOG.error(msg)
|
||||
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
|
||||
request=req)
|
||||
|
||||
except exception.ImageSizeLimitExceeded as e:
|
||||
msg = _("The incoming image is too large: %") % e
|
||||
LOG.error(msg)
|
||||
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
|
||||
request=req)
|
||||
|
||||
except exception.StorageWriteDenied as e:
|
||||
msg = _("Insufficient permissions on image storage media: %s") % e
|
||||
LOG.error(msg)
|
||||
|
||||
@ -122,6 +122,12 @@ class ImagesController(object):
|
||||
raise webob.exc.HTTPNotFound(explanation=msg)
|
||||
except exception.Forbidden as e:
|
||||
raise webob.exc.HTTPForbidden(explanation=unicode(e))
|
||||
except exception.StorageQuotaFull as e:
|
||||
msg = (_("Denying attempt to upload image because it exceeds the ."
|
||||
"quota: %s") % e)
|
||||
LOG.info(msg)
|
||||
raise webob.exc.HTTPRequestEntityTooLarge(
|
||||
explanation=msg, request=req, content_type='text/plain')
|
||||
|
||||
return image
|
||||
|
||||
|
||||
@ -65,6 +65,10 @@ common_opts = [
|
||||
cfg.IntOpt('image_size_cap', default=1099511627776,
|
||||
help=_("Maximum size of image a user can upload in bytes. "
|
||||
"Defaults to 1099511627776 bytes (1 TB).")),
|
||||
cfg.IntOpt('user_storage_quota', default=0,
|
||||
help=_("Set a system wide quota for every user. This value is "
|
||||
"the total number of bytes that a user can use across "
|
||||
"all storage systems. A value of 0 means unlimited.")),
|
||||
cfg.BoolOpt('enable_v1_api', default=True,
|
||||
help=_("Deploy the v1 OpenStack Images API. ")),
|
||||
cfg.BoolOpt('enable_v2_api', default=True,
|
||||
|
||||
@ -86,6 +86,11 @@ class StorageFull(GlanceException):
|
||||
message = _("There is not enough disk space on the image storage media.")
|
||||
|
||||
|
||||
class StorageQuotaFull(GlanceException):
|
||||
message = _("The size of the data %(image_size)s will exceed the limit. "
|
||||
"%(remaining)s bytes remaining.")
|
||||
|
||||
|
||||
class StorageWriteDenied(GlanceException):
|
||||
message = _("Permission to write image storage media denied.")
|
||||
|
||||
|
||||
@ -216,3 +216,8 @@ def image_tag_delete(client, image_id, value, session=None):
|
||||
def image_tag_get_all(client, image_id, session=None):
|
||||
"""Get a list of tags for a specific image."""
|
||||
return client.image_tag_get_all(image_id=image_id)
|
||||
|
||||
|
||||
@_get_client
|
||||
def user_get_storage_usage(client, owner_id, image_id=None, session=None):
|
||||
return client.user_get_storage_usage(owner_id=owner_id, image_id=image_id)
|
||||
|
||||
@ -648,3 +648,12 @@ def is_image_visible(context, image, status=None):
|
||||
|
||||
# Private image
|
||||
return False
|
||||
|
||||
|
||||
def user_get_storage_usage(context, owner_id, image_id=None, session=None):
|
||||
images = image_get_all(context, filters={'owner': owner_id})
|
||||
total = 0
|
||||
for image in images:
|
||||
if image['id'] != image_id:
|
||||
total = total + (image['size'] * len(image['locations']))
|
||||
return total
|
||||
|
||||
@ -688,6 +688,17 @@ def _drop_protected_attrs(model_class, values):
|
||||
del values[attr]
|
||||
|
||||
|
||||
def _image_get_disk_usage_by_owner(owner, session, image_id=None):
|
||||
query = session.query(models.Image)
|
||||
query = query.filter(models.Image.owner == owner)
|
||||
if image_id is not None:
|
||||
query = query.filter(models.Image.id != image_id)
|
||||
query = query.filter(models.Image.size > 0)
|
||||
images = query.all()
|
||||
total = sum([i.size * len(i.locations) for i in images])
|
||||
return total
|
||||
|
||||
|
||||
def _validate_image(values):
|
||||
"""
|
||||
Validates the incoming data and raises a Invalid exception
|
||||
@ -695,7 +706,6 @@ def _validate_image(values):
|
||||
|
||||
:param values: Mapping of image metadata to check
|
||||
"""
|
||||
status = values.get('status')
|
||||
|
||||
status = values.get('status', None)
|
||||
if not status:
|
||||
@ -1093,3 +1103,10 @@ def image_tag_get_all(context, image_id, session=None):
|
||||
.order_by(sqlalchemy.asc(models.ImageTag.created_at))\
|
||||
.all()
|
||||
return [tag['value'] for tag in tags]
|
||||
|
||||
|
||||
def user_get_storage_usage(context, owner_id, image_id=None, session=None):
|
||||
session = session or _get_session()
|
||||
total_size = _image_get_disk_usage_by_owner(
|
||||
owner_id, session, image_id=image_id)
|
||||
return total_size
|
||||
|
||||
@ -18,6 +18,7 @@ from glance.api import policy
|
||||
import glance.db
|
||||
import glance.domain
|
||||
import glance.notifier
|
||||
import glance.quota
|
||||
import glance.store
|
||||
|
||||
|
||||
@ -34,8 +35,10 @@ class Gateway(object):
|
||||
image_factory = glance.domain.ImageFactory()
|
||||
store_image_factory = glance.store.ImageFactoryProxy(
|
||||
image_factory, context, self.store_api)
|
||||
quota_image_factory = glance.quota.ImageFactoryProxy(
|
||||
store_image_factory, context, self.db_api)
|
||||
policy_image_factory = policy.ImageFactoryProxy(
|
||||
store_image_factory, context, self.policy)
|
||||
quota_image_factory, context, self.policy)
|
||||
notifier_image_factory = glance.notifier.ImageFactoryProxy(
|
||||
policy_image_factory, context, self.notifier)
|
||||
authorized_image_factory = authorization.ImageFactoryProxy(
|
||||
@ -54,8 +57,10 @@ class Gateway(object):
|
||||
image_repo = glance.db.ImageRepo(context, self.db_api)
|
||||
store_image_repo = glance.store.ImageRepoProxy(
|
||||
image_repo, context, self.store_api)
|
||||
quota_image_repo = glance.quota.ImageRepoProxy(
|
||||
store_image_repo, context, self.db_api)
|
||||
policy_image_repo = policy.ImageRepoProxy(
|
||||
store_image_repo, context, self.policy)
|
||||
quota_image_repo, context, self.policy)
|
||||
notifier_image_repo = glance.notifier.ImageRepoProxy(
|
||||
policy_image_repo, context, self.notifier)
|
||||
authorized_image_repo = authorization.ImageRepoProxy(
|
||||
|
||||
174
glance/quota/__init__.py
Normal file
174
glance/quota/__init__.py
Normal file
@ -0,0 +1,174 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2013, Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
import glance.api.common
|
||||
import glance.common.exception as exception
|
||||
from glance.common import utils
|
||||
import glance.domain
|
||||
import glance.domain.proxy
|
||||
import glance.openstack.common.log as logging
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ImageRepoProxy(glance.domain.proxy.Repo):
|
||||
|
||||
def __init__(self, image_repo, context, db_api):
|
||||
self.image_repo = image_repo
|
||||
self.db_api = db_api
|
||||
proxy_kwargs = {'db_api': db_api, 'context': context}
|
||||
super(ImageRepoProxy, self).__init__(image_repo,
|
||||
item_proxy_class=ImageProxy,
|
||||
item_proxy_kwargs=proxy_kwargs)
|
||||
|
||||
|
||||
class ImageFactoryProxy(glance.domain.proxy.ImageFactory):
|
||||
def __init__(self, factory, context, db_api):
|
||||
proxy_kwargs = {'db_api': db_api, 'context': context}
|
||||
super(ImageFactoryProxy, self).__init__(factory,
|
||||
proxy_class=ImageProxy,
|
||||
proxy_kwargs=proxy_kwargs)
|
||||
|
||||
|
||||
class QuotaImageLocationsProxy(object):
|
||||
|
||||
def __init__(self, image, context, db_api):
|
||||
self.image = image
|
||||
self.context = context
|
||||
self.db_api = db_api
|
||||
self.locations = image.locations
|
||||
|
||||
def __cast__(self, *args, **kwargs):
|
||||
return self.locations.__cast__(*args, **kwargs)
|
||||
|
||||
def __contains__(self, *args, **kwargs):
|
||||
return self.locations.__contains__(*args, **kwargs)
|
||||
|
||||
def __delitem__(self, *args, **kwargs):
|
||||
return self.locations.__delitem__(*args, **kwargs)
|
||||
|
||||
def __delslice__(self, *args, **kwargs):
|
||||
return self.locations.__delslice__(*args, **kwargs)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.locations == other
|
||||
|
||||
def __getitem__(self, *args, **kwargs):
|
||||
return self.locations.__getitem__(*args, **kwargs)
|
||||
|
||||
def __iadd__(self, other):
|
||||
if not hasattr(other, '__iter__'):
|
||||
raise TypeError()
|
||||
self._check_quota(len(list(other)))
|
||||
return self.locations.__iadd__(other)
|
||||
|
||||
def __iter__(self, *args, **kwargs):
|
||||
return self.locations.__iter__(*args, **kwargs)
|
||||
|
||||
def __len__(self, *args, **kwargs):
|
||||
return self.locations.__len__(*args, **kwargs)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
return self.locations.__setitem__(key, value)
|
||||
|
||||
def count(self, *args, **kwargs):
|
||||
return self.locations.count(*args, **kwargs)
|
||||
|
||||
def index(self, *args, **kwargs):
|
||||
return self.locations.index(*args, **kwargs)
|
||||
|
||||
def pop(self, *args, **kwargs):
|
||||
return self.locations.pop(*args, **kwargs)
|
||||
|
||||
def remove(self, *args, **kwargs):
|
||||
return self.locations.remove(*args, **kwargs)
|
||||
|
||||
def reverse(self, *args, **kwargs):
|
||||
return self.locations.reverse(*args, **kwargs)
|
||||
|
||||
def __getitem__(self, *args, **kwargs):
|
||||
return self.locations.__getitem__(*args, **kwargs)
|
||||
|
||||
def _check_quota(self, count):
|
||||
glance.api.common.check_quota(
|
||||
self.context, self.image.size * count, self.db_api)
|
||||
|
||||
def append(self, object):
|
||||
self._check_quota(1)
|
||||
return self.locations.append(object)
|
||||
|
||||
def insert(self, index, object):
|
||||
self._check_quota(1)
|
||||
return self.locations.insert(index, object)
|
||||
|
||||
def extend(self, iter):
|
||||
self._check_quota(len(list(iter)))
|
||||
return self.locations.extend(iter)
|
||||
|
||||
|
||||
class ImageProxy(glance.domain.proxy.Image):
|
||||
|
||||
def __init__(self, image, context, db_api):
|
||||
self.image = image
|
||||
self.context = context
|
||||
self.db_api = db_api
|
||||
super(ImageProxy, self).__init__(image)
|
||||
|
||||
def set_data(self, data, size=None):
|
||||
remaining = glance.api.common.check_quota(
|
||||
self.context, size, self.db_api, image_id=self.image.image_id)
|
||||
if remaining is not None:
|
||||
# NOTE(jbresnah) we are trying to enforce a quota, put a limit
|
||||
# reader on the data
|
||||
data = utils.LimitingReader(data, remaining)
|
||||
try:
|
||||
self.image.set_data(data, size=size)
|
||||
except exception.ImageSizeLimitExceeded as ex:
|
||||
raise exception.StorageQuotaFull(image_size=size,
|
||||
remaining=remaining)
|
||||
|
||||
# NOTE(jbresnah) If two uploads happen at the same time and neither
|
||||
# properly sets the size attribute than there is a race condition
|
||||
# that will allow for the quota to be broken. Thus we must recheck
|
||||
# the quota after the upload and thus after we know the size
|
||||
try:
|
||||
glance.api.common.check_quota(
|
||||
self.context, self.image.size, self.db_api,
|
||||
image_id=self.image.image_id)
|
||||
except exception.StorageQuotaFull:
|
||||
LOG.info(_('Cleaning up %s after exceeding the quota.')
|
||||
% self.image.image_id)
|
||||
location = self.image.locations[0]['url']
|
||||
glance.store.safe_delete_from_backend(
|
||||
location, self.context, self.image.image_id)
|
||||
raise
|
||||
|
||||
@property
|
||||
def locations(self):
|
||||
return QuotaImageLocationsProxy(self.image,
|
||||
self.context,
|
||||
self.db_api)
|
||||
|
||||
@locations.setter
|
||||
def locations(self, value):
|
||||
if not isinstance(value, (list, QuotaImageLocationsProxy)):
|
||||
raise exception.Invalid(_('Invalid locations: %s') % value)
|
||||
glance.api.common.check_quota(
|
||||
self.context, self.image.size * len(value), self.db_api,
|
||||
image_id=self.image.image_id)
|
||||
self.image.locations = value
|
||||
@ -314,6 +314,7 @@ class ApiServer(Server):
|
||||
default_sql_connection = 'sqlite:////%s/tests.sqlite' % self.test_dir
|
||||
self.sql_connection = os.environ.get('GLANCE_TEST_SQL_CONNECTION',
|
||||
default_sql_connection)
|
||||
self.user_storage_quota = 0
|
||||
|
||||
self.conf_base = """[DEFAULT]
|
||||
verbose = %(verbose)s
|
||||
@ -360,6 +361,7 @@ db_auto_create = False
|
||||
sql_connection = %(sql_connection)s
|
||||
show_image_direct_url = %(show_image_direct_url)s
|
||||
show_multiple_locations = %(show_multiple_locations)s
|
||||
user_storage_quota = %(user_storage_quota)s
|
||||
enable_v1_api = %(enable_v1_api)s
|
||||
enable_v2_api= %(enable_v2_api)s
|
||||
[paste_deploy]
|
||||
@ -448,6 +450,8 @@ class RegistryServer(Server):
|
||||
self.owner_is_tenant = True
|
||||
self.workers = 0
|
||||
self.api_version = 1
|
||||
self.user_storage_quota = 0
|
||||
|
||||
self.conf_base = """[DEFAULT]
|
||||
verbose = %(verbose)s
|
||||
debug = %(debug)s
|
||||
@ -461,6 +465,7 @@ api_limit_max = 1000
|
||||
limit_param_default = 25
|
||||
owner_is_tenant = %(owner_is_tenant)s
|
||||
workers = %(workers)s
|
||||
user_storage_quota = %(user_storage_quota)s
|
||||
[paste_deploy]
|
||||
flavor = %(deployment_flavor)s
|
||||
"""
|
||||
|
||||
@ -1092,6 +1092,79 @@ class DriverTests(object):
|
||||
self.assertEqual(0, len(self.db_api.image_member_find(self.context)))
|
||||
|
||||
|
||||
class DriverQuotaTests(test_utils.BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(DriverQuotaTests, self).setUp()
|
||||
self.owner_id1 = uuidutils.generate_uuid()
|
||||
self.context1 = context.RequestContext(
|
||||
is_admin=False, auth_tok='user:user:user', user=self.owner_id1)
|
||||
self.db_api = db_tests.get_db(self.config)
|
||||
db_tests.reset_db(self.db_api)
|
||||
self.addCleanup(timeutils.clear_time_override)
|
||||
dt1 = timeutils.utcnow()
|
||||
dt2 = dt1 + datetime.timedelta(microseconds=5)
|
||||
fixtures = [
|
||||
{
|
||||
'id': UUID1,
|
||||
'created_at': dt1,
|
||||
'updated_at': dt1,
|
||||
'size': 13,
|
||||
'owner': self.owner_id1,
|
||||
},
|
||||
{
|
||||
'id': UUID2,
|
||||
'created_at': dt1,
|
||||
'updated_at': dt2,
|
||||
'size': 17,
|
||||
'owner': self.owner_id1,
|
||||
},
|
||||
{
|
||||
'id': UUID3,
|
||||
'created_at': dt2,
|
||||
'updated_at': dt2,
|
||||
'size': 7,
|
||||
'owner': self.owner_id1,
|
||||
},
|
||||
]
|
||||
self.owner1_fixtures = [
|
||||
build_image_fixture(**fixture) for fixture in fixtures]
|
||||
|
||||
for fixture in self.owner1_fixtures:
|
||||
self.db_api.image_create(self.context1, fixture)
|
||||
|
||||
def test_storage_quota(self):
|
||||
total = reduce(lambda x, y: x + y,
|
||||
[f['size'] for f in self.owner1_fixtures])
|
||||
x = self.db_api.user_get_storage_usage(self.context1, self.owner_id1)
|
||||
self.assertEqual(total, x)
|
||||
|
||||
def test_storage_quota_without_image_id(self):
|
||||
total = reduce(lambda x, y: x + y,
|
||||
[f['size'] for f in self.owner1_fixtures])
|
||||
total = total - self.owner1_fixtures[0]['size']
|
||||
x = self.db_api.user_get_storage_usage(
|
||||
self.context1, self.owner_id1,
|
||||
image_id=self.owner1_fixtures[0]['id'])
|
||||
self.assertEqual(total, x)
|
||||
|
||||
def test_storage_quota_multiple_locations(self):
|
||||
dt1 = timeutils.utcnow()
|
||||
sz = 53
|
||||
new_fixture_dict = {'id': 'SOMEID', 'created_at': dt1,
|
||||
'updated_at': dt1, 'size': sz,
|
||||
'owner': self.owner_id1}
|
||||
new_fixture = build_image_fixture(**new_fixture_dict)
|
||||
new_fixture['locations'].append({'url': 'file:///some/path/file',
|
||||
'metadata': {}})
|
||||
self.db_api.image_create(self.context1, new_fixture)
|
||||
|
||||
total = reduce(lambda x, y: x + y,
|
||||
[f['size'] for f in self.owner1_fixtures]) + (sz * 2)
|
||||
x = self.db_api.user_get_storage_usage(self.context1, self.owner_id1)
|
||||
self.assertEqual(total, x)
|
||||
|
||||
|
||||
class TestVisibility(test_utils.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestVisibility, self).setUp()
|
||||
|
||||
@ -72,3 +72,15 @@ class TestRegistryDriver(base.TestDriver,
|
||||
def tearDown(self):
|
||||
self.registry_server.stop()
|
||||
super(TestRegistryDriver, self).tearDown()
|
||||
|
||||
|
||||
class TestRegistryQuota(base.DriverQuotaTests, FunctionalInitWrapper):
|
||||
|
||||
def setUp(self):
|
||||
db_tests.load(get_db, reset_db)
|
||||
super(TestRegistryQuota, self).setUp()
|
||||
self.addCleanup(db_tests.reset)
|
||||
|
||||
def tearDown(self):
|
||||
self.registry_server.stop()
|
||||
super(TestRegistryQuota, self).tearDown()
|
||||
|
||||
@ -35,6 +35,14 @@ class TestSimpleDriver(base.TestDriver, base.DriverTests):
|
||||
self.addCleanup(db_tests.reset)
|
||||
|
||||
|
||||
class TestSimpleQuota(base.DriverQuotaTests):
|
||||
|
||||
def setUp(self):
|
||||
db_tests.load(get_db, reset_db)
|
||||
super(TestSimpleQuota, self).setUp()
|
||||
self.addCleanup(db_tests.reset)
|
||||
|
||||
|
||||
class TestSimpleVisibility(base.TestVisibility, base.VisibilityTests):
|
||||
|
||||
def setUp(self):
|
||||
|
||||
@ -1111,3 +1111,86 @@ class TestImageMembers(functional.FunctionalTest):
|
||||
self.assertEqual(404, response.status_code)
|
||||
|
||||
self.stop_servers()
|
||||
|
||||
|
||||
class TestQuotas(functional.FunctionalTest):
|
||||
|
||||
def setUp(self):
|
||||
super(TestQuotas, self).setUp()
|
||||
self.cleanup()
|
||||
self.api_server.deployment_flavor = 'noauth'
|
||||
self.user_storage_quota = 100
|
||||
self.start_servers(**self.__dict__.copy())
|
||||
|
||||
def _url(self, path):
|
||||
return 'http://127.0.0.1:%d%s' % (self.api_port, path)
|
||||
|
||||
def _headers(self, custom_headers=None):
|
||||
base_headers = {
|
||||
'X-Identity-Status': 'Confirmed',
|
||||
'X-Auth-Token': '932c5c84-02ac-4fe5-a9ba-620af0e2bb96',
|
||||
'X-User-Id': 'f9a41d13-0c13-47e9-bee2-ce4e8bfe958e',
|
||||
'X-Tenant-Id': TENANT1,
|
||||
'X-Roles': 'member',
|
||||
}
|
||||
base_headers.update(custom_headers or {})
|
||||
return base_headers
|
||||
|
||||
def test_image_upload_under_quota(self):
|
||||
# Image list should be empty
|
||||
path = self._url('/v2/images')
|
||||
response = requests.get(path, headers=self._headers())
|
||||
self.assertEqual(200, response.status_code)
|
||||
images = json.loads(response.text)['images']
|
||||
self.assertEqual(0, len(images))
|
||||
|
||||
# Create an image (with a deployer-defined property)
|
||||
path = self._url('/v2/images')
|
||||
headers = self._headers({'content-type': 'application/json'})
|
||||
data = json.dumps({'name': 'image-2',
|
||||
'disk_format': 'aki', 'container_format': 'aki'})
|
||||
response = requests.post(path, headers=headers, data=data)
|
||||
self.assertEqual(201, response.status_code)
|
||||
image = json.loads(response.text)
|
||||
image_id = image['id']
|
||||
|
||||
# upload data
|
||||
data = 'x' * (self.user_storage_quota - 1)
|
||||
path = self._url('/v2/images/%s/file' % image_id)
|
||||
headers = self._headers({'Content-Type': 'application/octet-stream'})
|
||||
response = requests.put(path, headers=headers, data=data)
|
||||
self.assertEqual(204, response.status_code)
|
||||
|
||||
# Deletion should work
|
||||
path = self._url('/v2/images/%s' % image_id)
|
||||
response = requests.delete(path, headers=self._headers())
|
||||
self.assertEqual(204, response.status_code)
|
||||
|
||||
def test_image_upload_exceed_quota(self):
|
||||
# Image list should be empty
|
||||
path = self._url('/v2/images')
|
||||
response = requests.get(path, headers=self._headers())
|
||||
self.assertEqual(200, response.status_code)
|
||||
images = json.loads(response.text)['images']
|
||||
self.assertEqual(0, len(images))
|
||||
|
||||
# Create an image (with a deployer-defined property)
|
||||
path = self._url('/v2/images')
|
||||
headers = self._headers({'content-type': 'application/json'})
|
||||
data = json.dumps({'name': 'image-1', 'type': 'kernel', 'foo': 'bar',
|
||||
'disk_format': 'aki', 'container_format': 'aki'})
|
||||
response = requests.post(path, headers=headers, data=data)
|
||||
self.assertEqual(201, response.status_code)
|
||||
image = json.loads(response.text)
|
||||
image_id = image['id']
|
||||
|
||||
# upload data
|
||||
data = 'x' * (self.user_storage_quota + 1)
|
||||
path = self._url('/v2/images/%s/file' % image_id)
|
||||
headers = self._headers({'Content-Type': 'application/octet-stream'})
|
||||
response = requests.put(path, headers=headers, data=data)
|
||||
self.assertEqual(413, response.status_code)
|
||||
|
||||
path = self._url('/v2/images/%s' % image_id)
|
||||
response = requests.delete(path, headers=self._headers())
|
||||
self.assertEqual(204, response.status_code)
|
||||
|
||||
259
glance/tests/unit/test_quota.py
Normal file
259
glance/tests/unit/test_quota.py
Normal file
@ -0,0 +1,259 @@
|
||||
# Copyright 2013, Red Hat, Inc.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the 'License'); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import mox
|
||||
|
||||
from glance.common import exception
|
||||
import glance.quota
|
||||
import glance.store
|
||||
from glance.tests.unit import utils as unit_test_utils
|
||||
from glance.tests import utils as test_utils
|
||||
|
||||
UUID1 = 'c80a1a6c-bd1f-41c5-90ee-81afedb1d58d'
|
||||
|
||||
|
||||
class ImageRepoStub(object):
|
||||
def get(self, *args, **kwargs):
|
||||
return 'image_from_get'
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
return 'image_from_save'
|
||||
|
||||
def add(self, *args, **kwargs):
|
||||
return 'image_from_add'
|
||||
|
||||
def list(self, *args, **kwargs):
|
||||
return ['image_from_list_0', 'image_from_list_1']
|
||||
|
||||
|
||||
class ImageStub(object):
|
||||
def __init__(self, image_id, visibility='private'):
|
||||
self.image_id = image_id
|
||||
self.visibility = visibility
|
||||
self.status = 'active'
|
||||
|
||||
def delete(self):
|
||||
self.status = 'deleted'
|
||||
|
||||
|
||||
class ImageFactoryStub(object):
|
||||
def new_image(self, image_id=None, name=None, visibility='private',
|
||||
min_disk=0, min_ram=0, protected=False, owner=None,
|
||||
disk_format=None, container_format=None,
|
||||
extra_properties=None, tags=None, **other_args):
|
||||
self.visibility = visibility
|
||||
return 'new_image'
|
||||
|
||||
|
||||
class FakeContext(object):
|
||||
owner = 'someone'
|
||||
is_admin = False
|
||||
|
||||
|
||||
class FakeImage(object):
|
||||
size = None
|
||||
image_id = 'someid'
|
||||
locations = [{'url': 'file:///not/a/path', 'metadata': {}}]
|
||||
|
||||
def set_data(self, data, size=None):
|
||||
self.size = 0
|
||||
for d in data:
|
||||
self.size = self. size + len(d)
|
||||
|
||||
|
||||
class TestImageQuota(test_utils.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestImageQuota, self).setUp()
|
||||
self.mox = mox.Mox()
|
||||
|
||||
def tearDown(self):
|
||||
super(TestImageQuota, self).tearDown()
|
||||
self.mox.UnsetStubs()
|
||||
|
||||
def _get_image(self, location_count=1, image_size=10):
|
||||
context = FakeContext()
|
||||
db_api = unit_test_utils.FakeDB()
|
||||
base_image = FakeImage()
|
||||
base_image.image_id = 'xyz'
|
||||
base_image.size = image_size
|
||||
image = glance.quota.ImageProxy(base_image, context, db_api)
|
||||
locations = []
|
||||
for i in range(location_count):
|
||||
locations.append({'url': 'file:///g/there/it/is%d' % i,
|
||||
'metadata': {}})
|
||||
image_values = {'id': 'xyz', 'owner': context.owner,
|
||||
'status': 'active', 'size': image_size,
|
||||
'locations': locations}
|
||||
db_api.image_create(context, image_values)
|
||||
return image
|
||||
|
||||
def test_quota_allowed(self):
|
||||
quota = 10
|
||||
self.config(user_storage_quota=quota)
|
||||
context = FakeContext()
|
||||
db_api = unit_test_utils.FakeDB()
|
||||
base_image = FakeImage()
|
||||
base_image.image_id = 'id'
|
||||
image = glance.quota.ImageProxy(base_image, context, db_api)
|
||||
data = '*' * quota
|
||||
base_image.set_data(data, size=None)
|
||||
image.set_data(data)
|
||||
self.assertEqual(quota, base_image.size)
|
||||
|
||||
def _quota_exceeded_size(self, quota, data,
|
||||
deleted=True, size=None):
|
||||
self.config(user_storage_quota=quota)
|
||||
context = FakeContext()
|
||||
db_api = unit_test_utils.FakeDB()
|
||||
base_image = FakeImage()
|
||||
base_image.image_id = 'id'
|
||||
image = glance.quota.ImageProxy(base_image, context, db_api)
|
||||
|
||||
if deleted:
|
||||
self.mox.StubOutWithMock(glance.store, 'safe_delete_from_backend')
|
||||
glance.store.safe_delete_from_backend(
|
||||
base_image.locations[0]['url'],
|
||||
context,
|
||||
image.image_id)
|
||||
|
||||
self.mox.ReplayAll()
|
||||
self.assertRaises(exception.StorageQuotaFull,
|
||||
image.set_data,
|
||||
data,
|
||||
size=size)
|
||||
self.mox.VerifyAll()
|
||||
|
||||
def test_quota_exceeded_no_size(self):
|
||||
quota = 10
|
||||
data = '*' * (quota + 1)
|
||||
self._quota_exceeded_size(quota, data)
|
||||
|
||||
def test_quota_exceeded_with_right_size(self):
|
||||
quota = 10
|
||||
data = '*' * (quota + 1)
|
||||
self._quota_exceeded_size(quota, data, size=len(data), deleted=False)
|
||||
|
||||
def test_quota_exceeded_with_lie_size(self):
|
||||
quota = 10
|
||||
data = '*' * (quota + 1)
|
||||
self._quota_exceeded_size(quota, data, deleted=False, size=quota - 1)
|
||||
|
||||
def test_append_location(self):
|
||||
new_location = {'url': 'file:///a/path', 'metadata': {}}
|
||||
image = self._get_image()
|
||||
pre_add_locations = image.locations[:]
|
||||
image.locations.append(new_location)
|
||||
pre_add_locations.append(new_location)
|
||||
self.assertEqual(image.locations, pre_add_locations)
|
||||
|
||||
def test_insert_location(self):
|
||||
new_location = {'url': 'file:///a/path', 'metadata': {}}
|
||||
image = self._get_image()
|
||||
pre_add_locations = image.locations[:]
|
||||
image.locations.insert(0, new_location)
|
||||
pre_add_locations.insert(0, new_location)
|
||||
self.assertEqual(image.locations, pre_add_locations)
|
||||
|
||||
def test_extend_location(self):
|
||||
new_location = {'url': 'file:///a/path', 'metadata': {}}
|
||||
image = self._get_image()
|
||||
pre_add_locations = image.locations[:]
|
||||
image.locations.extend([new_location])
|
||||
pre_add_locations.extend([new_location])
|
||||
self.assertEqual(image.locations, pre_add_locations)
|
||||
|
||||
def test_iadd_location(self):
|
||||
new_location = {'url': 'file:///a/path', 'metadata': {}}
|
||||
image = self._get_image()
|
||||
pre_add_locations = image.locations[:]
|
||||
image.locations += [new_location]
|
||||
pre_add_locations += [new_location]
|
||||
self.assertEqual(image.locations, pre_add_locations)
|
||||
|
||||
def test_set_location(self):
|
||||
new_location = {'url': 'file:///a/path', 'metadata': {}}
|
||||
image = self._get_image()
|
||||
image.locations = [new_location]
|
||||
self.assertEqual(image.locations, [new_location])
|
||||
|
||||
def test_exceed_append_location(self):
|
||||
image_size = 10
|
||||
max_images = 2
|
||||
quota = image_size * max_images
|
||||
self.config(user_storage_quota=quota)
|
||||
image = self._get_image(image_size=image_size,
|
||||
location_count=max_images)
|
||||
self.assertRaises(exception.StorageQuotaFull,
|
||||
image.locations.append,
|
||||
{'url': 'file:///a/path', 'metadata': {}})
|
||||
|
||||
def test_exceed_append_location(self):
|
||||
image_size = 10
|
||||
max_images = 2
|
||||
quota = image_size * max_images
|
||||
self.config(user_storage_quota=quota)
|
||||
image = self._get_image(image_size=image_size,
|
||||
location_count=max_images)
|
||||
self.assertRaises(exception.StorageQuotaFull,
|
||||
image.locations.insert,
|
||||
0,
|
||||
{'url': 'file:///a/path', 'metadata': {}})
|
||||
|
||||
def test_exceed_extend_location(self):
|
||||
image_size = 10
|
||||
max_images = 2
|
||||
quota = image_size * max_images
|
||||
self.config(user_storage_quota=quota)
|
||||
image = self._get_image(image_size=image_size,
|
||||
location_count=max_images)
|
||||
self.assertRaises(exception.StorageQuotaFull,
|
||||
image.locations.extend,
|
||||
[{'url': 'file:///a/path', 'metadata': {}}])
|
||||
|
||||
def test_set_location_under(self):
|
||||
image_size = 10
|
||||
max_images = 1
|
||||
quota = image_size * max_images
|
||||
self.config(user_storage_quota=quota)
|
||||
image = self._get_image(image_size=image_size,
|
||||
location_count=max_images)
|
||||
image.locations = [{'url': 'file:///a/path', 'metadata': {}}]
|
||||
|
||||
def test_set_location_exceed(self):
|
||||
image_size = 10
|
||||
max_images = 1
|
||||
quota = image_size * max_images
|
||||
self.config(user_storage_quota=quota)
|
||||
image = self._get_image(image_size=image_size,
|
||||
location_count=max_images)
|
||||
try:
|
||||
image.locations = [{'url': 'file:///a/path', 'metadata': {}},
|
||||
{'url': 'file:///a/path2', 'metadata': {}}]
|
||||
self.fail('Should have raised the quota exception')
|
||||
except exception.StorageQuotaFull:
|
||||
pass
|
||||
|
||||
def test_iadd_location_exceed(self):
|
||||
image_size = 10
|
||||
max_images = 1
|
||||
quota = image_size * max_images
|
||||
self.config(user_storage_quota=quota)
|
||||
image = self._get_image(image_size=image_size,
|
||||
location_count=max_images)
|
||||
try:
|
||||
image.locations += [{'url': 'file:///a/path', 'metadata': {}}]
|
||||
self.fail('Should have raised the quota exception')
|
||||
except exception.StorageQuotaFull:
|
||||
pass
|
||||
@ -416,6 +416,74 @@ class TestGlanceAPI(base.IsolatedUnitTest):
|
||||
res = req.get_response(self.api)
|
||||
self.assertEquals(res.status_int, 400)
|
||||
|
||||
def test_add_image_size_header_exceed_quota(self):
|
||||
quota = 500
|
||||
self.config(user_storage_quota=quota)
|
||||
fixture_headers = {'x-image-meta-size': quota + 1,
|
||||
'x-image-meta-name': 'fake image #3',
|
||||
'x-image-meta-container_format': 'bare',
|
||||
'x-image-meta-disk_format': 'qcow2',
|
||||
'content-type': 'application/octet-stream',
|
||||
}
|
||||
|
||||
req = webob.Request.blank("/images")
|
||||
req.method = 'POST'
|
||||
for k, v in fixture_headers.iteritems():
|
||||
req.headers[k] = v
|
||||
req.body = 'X' * (quota + 1)
|
||||
res = req.get_response(self.api)
|
||||
self.assertEquals(res.status_int, 413)
|
||||
|
||||
def test_add_image_size_data_exceed_quota(self):
|
||||
quota = 500
|
||||
self.config(user_storage_quota=quota)
|
||||
fixture_headers = {
|
||||
'x-image-meta-name': 'fake image #3',
|
||||
'x-image-meta-container_format': 'bare',
|
||||
'x-image-meta-disk_format': 'qcow2',
|
||||
'content-type': 'application/octet-stream',
|
||||
}
|
||||
|
||||
req = webob.Request.blank("/images")
|
||||
req.method = 'POST'
|
||||
|
||||
req.body = 'X' * (quota + 1)
|
||||
for k, v in fixture_headers.iteritems():
|
||||
req.headers[k] = v
|
||||
|
||||
res = req.get_response(self.api)
|
||||
self.assertEquals(res.status_int, 413)
|
||||
|
||||
def test_add_image_size_data_exceed_quota_readd(self):
|
||||
quota = 500
|
||||
self.config(user_storage_quota=quota)
|
||||
fixture_headers = {
|
||||
'x-image-meta-name': 'fake image #3',
|
||||
'x-image-meta-container_format': 'bare',
|
||||
'x-image-meta-disk_format': 'qcow2',
|
||||
'content-type': 'application/octet-stream',
|
||||
}
|
||||
|
||||
req = webob.Request.blank("/images")
|
||||
req.method = 'POST'
|
||||
req.body = 'X' * (quota + 1)
|
||||
for k, v in fixture_headers.iteritems():
|
||||
req.headers[k] = v
|
||||
|
||||
res = req.get_response(self.api)
|
||||
self.assertEquals(res.status_int, 413)
|
||||
|
||||
used_size = sum([f['size'] for f in self.FIXTURES])
|
||||
|
||||
req = webob.Request.blank("/images")
|
||||
req.method = 'POST'
|
||||
req.body = 'X' * (quota - used_size)
|
||||
for k, v in fixture_headers.iteritems():
|
||||
req.headers[k] = v
|
||||
|
||||
res = req.get_response(self.api)
|
||||
self.assertEquals(res.status_int, 201)
|
||||
|
||||
def _add_check_no_url_info(self):
|
||||
|
||||
fixture_headers = {'x-image-meta-disk-format': 'ami',
|
||||
|
||||
Loading…
Reference in New Issue
Block a user