Merge "Basic container quotas"
This commit is contained in:
Jenkins
@@ -179,3 +179,10 @@ Bulk Operations (Delete and Archive Auto Extraction)
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Container Quotas
|
||||
=============
|
||||
|
||||
.. automodule:: swift.common.middleware.container_quotas
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
# eventlet_debug = false
|
||||
|
||||
[pipeline:main]
|
||||
pipeline = catch_errors healthcheck cache ratelimit tempauth proxy-logging proxy-server
|
||||
pipeline = catch_errors healthcheck cache ratelimit tempauth container-quotas proxy-logging proxy-server
|
||||
|
||||
[app:proxy-server]
|
||||
use = egg:swift#proxy
|
||||
@@ -338,3 +338,7 @@ use = egg:swift#bulk
|
||||
# max_containers_per_extraction = 10000
|
||||
# max_failed_files = 1000
|
||||
# max_deletes_per_request = 1000
|
||||
|
||||
# Note: Put after auth in the pipeline.
|
||||
[filter:container-quotas]
|
||||
use = egg:swift#container_quotas
|
||||
|
||||
6
setup.py
6
setup.py
@@ -99,9 +99,11 @@ setup(
|
||||
'tempurl=swift.common.middleware.tempurl:filter_factory',
|
||||
'formpost=swift.common.middleware.formpost:filter_factory',
|
||||
'name_check=swift.common.middleware.name_check:filter_factory',
|
||||
'proxy_logging='
|
||||
'swift.common.middleware.proxy_logging:filter_factory',
|
||||
'bulk=swift.common.middleware.bulk:filter_factory',
|
||||
'container_quotas=swift.common.middleware.container_quotas:'
|
||||
'filter_factory',
|
||||
'proxy_logging=swift.common.middleware.proxy_logging:'
|
||||
'filter_factory',
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
107
swift/common/middleware/container_quotas.py
Normal file
107
swift/common/middleware/container_quotas.py
Normal file
@@ -0,0 +1,107 @@
|
||||
# Copyright (c) 2010-2012 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
The ``container_quotas`` middleware implements simple quotas that can be
|
||||
imposed on swift containers by a user with the ability to set container
|
||||
metadata, most likely the account administrator. This can be useful for
|
||||
limiting the scope of containers that are delegated to non-admin users, exposed
|
||||
to ``formpost`` uploads, or just as a self-imposed sanity check.
|
||||
|
||||
Any object PUT operations that exceed these quotas return a 413 response
|
||||
(request entity too large) with a descriptive body.
|
||||
|
||||
Quotas are subject to several limitations: eventual consistency, the timeliness
|
||||
of the cached container_info (60 second ttl by default), and it's unable to
|
||||
reject chunked transfer uploads that exceed the quota (though once the quota
|
||||
is exceeded, new chunked transfers will be refused).
|
||||
|
||||
Quotas are set by adding meta values to the container, and are validated when
|
||||
set:
|
||||
|
||||
+---------------------------------------------+-------------------------------+
|
||||
|Metadata | Use |
|
||||
+=============================================+===============================+
|
||||
| X-Container-Meta-Quota-Bytes | Maximum size of the |
|
||||
| | container, in bytes. |
|
||||
+---------------------------------------------+-------------------------------+
|
||||
| X-Container-Meta-Quota-Count | Maximum object count of the |
|
||||
| | container. |
|
||||
+---------------------------------------------+-------------------------------+
|
||||
"""
|
||||
|
||||
from swift.common.utils import split_path
|
||||
from swift.common.http import is_success
|
||||
from swift.proxy.controllers.base import get_container_info
|
||||
from swift.common.swob import Response, HTTPBadRequest, wsgify
|
||||
|
||||
|
||||
class ContainerQuotaMiddleware(object):
|
||||
def __init__(self, app, *args, **kwargs):
|
||||
self.app = app
|
||||
|
||||
def bad_response(self, req, container_info):
|
||||
# 401 if the user couldn't have PUT this object in the first place.
|
||||
# This prevents leaking the container's existence to unauthed users.
|
||||
if 'swift.authorize' in req.environ:
|
||||
req.acl = container_info['write_acl']
|
||||
aresp = req.environ['swift.authorize'](req)
|
||||
if aresp:
|
||||
return aresp
|
||||
return Response(status=413, body='Upload exceeds quota.')
|
||||
|
||||
@wsgify
|
||||
def __call__(self, req):
|
||||
try:
|
||||
(version, account, container, obj) = req.split_path(2, 4, True)
|
||||
except ValueError:
|
||||
return self.app
|
||||
|
||||
# verify new quota headers are properly formatted
|
||||
if container and not obj and req.method in ('PUT', 'POST'):
|
||||
val = req.headers.get('X-Container-Meta-Quota-Bytes')
|
||||
if val and not val.isdigit():
|
||||
return HTTPBadRequest(body='Invalid bytes quota.')
|
||||
val = req.headers.get('X-Container-Meta-Quota-Count')
|
||||
if val and not val.isdigit():
|
||||
return HTTPBadRequest(body='Invalid count quota.')
|
||||
|
||||
# check user uploads against quotas
|
||||
elif obj and req.method == 'PUT':
|
||||
container_info = get_container_info(req.environ, self.app)
|
||||
if not container_info or not is_success(container_info['status']):
|
||||
# this will hopefully 404 later
|
||||
return self.app
|
||||
if 'quota-bytes' in container_info.get('meta', {}) and \
|
||||
'bytes' in container_info and \
|
||||
container_info['meta']['quota-bytes'].isdigit():
|
||||
new_size = int(container_info['bytes']) + (req.content_length
|
||||
or 0)
|
||||
if int(container_info['meta']['quota-bytes']) < new_size:
|
||||
return self.bad_response(req, container_info)
|
||||
if 'quota-count' in container_info.get('meta', {}) and \
|
||||
'count' in container_info and \
|
||||
container_info['meta']['quota-count'].isdigit():
|
||||
new_count = int(container_info['count']) + 1
|
||||
if int(container_info['meta']['quota-count']) < new_count:
|
||||
return self.bad_response(req, container_info)
|
||||
|
||||
return self.app
|
||||
|
||||
|
||||
def filter_factory(global_conf, **local_conf):
|
||||
def container_quota_filter(app):
|
||||
return ContainerQuotaMiddleware(app)
|
||||
return container_quota_filter
|
||||
@@ -32,7 +32,9 @@ from eventlet import spawn_n, GreenPile
|
||||
from eventlet.queue import Queue, Empty, Full
|
||||
from eventlet.timeout import Timeout
|
||||
|
||||
from swift.common.utils import normalize_timestamp, config_true_value, public
|
||||
from swift.common.wsgi import make_pre_authed_request
|
||||
from swift.common.utils import normalize_timestamp, config_true_value, \
|
||||
public, split_path, cache_from_env
|
||||
from swift.common.bufferedhttp import http_connect
|
||||
from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH
|
||||
from swift.common.exceptions import ChunkReadTimeout, ConnectionTimeout
|
||||
@@ -188,6 +190,32 @@ def cors_validation(func):
|
||||
return wrapped
|
||||
|
||||
|
||||
def get_container_info(env, app):
|
||||
"""
|
||||
Get the info structure for a container, based on env and app.
|
||||
This is useful to middlewares.
|
||||
"""
|
||||
cache = cache_from_env(env)
|
||||
if not cache:
|
||||
return None
|
||||
(version, account, container, obj) = \
|
||||
split_path(env['PATH_INFO'], 2, 4, True)
|
||||
cache_key = get_container_memcache_key(account, container)
|
||||
# Use a unique environment cache key per container. If you copy this env
|
||||
# to make a new request, it won't accidentally reuse the old container info
|
||||
env_key = 'swift.%s' % cache_key
|
||||
if env_key not in env:
|
||||
container_info = cache.get(cache_key)
|
||||
if not container_info:
|
||||
resp = make_pre_authed_request(
|
||||
env, 'HEAD', '/%s/%s/%s' % (version, account, container)
|
||||
).get_response(app)
|
||||
container_info = headers_to_container_info(
|
||||
resp.headers, resp.status_int)
|
||||
env[env_key] = container_info
|
||||
return env[env_key]
|
||||
|
||||
|
||||
class Controller(object):
|
||||
"""Base WSGI controller class for the proxy"""
|
||||
server_type = 'Base'
|
||||
|
||||
165
test/unit/common/middleware/test_quotas.py
Normal file
165
test/unit/common/middleware/test_quotas.py
Normal file
@@ -0,0 +1,165 @@
|
||||
# Copyright (c) 2010-2012 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
|
||||
from swift.common.swob import Request, Response, HTTPUnauthorized
|
||||
from swift.common.middleware import container_quotas
|
||||
|
||||
class FakeCache(object):
|
||||
def __init__(self, val):
|
||||
if 'status' not in val:
|
||||
val['status'] = 200
|
||||
self.val = val
|
||||
|
||||
def get(self, *args):
|
||||
return self.val
|
||||
|
||||
class FakeApp(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __call__(self, env, start_response):
|
||||
start_response('200 OK', [])
|
||||
return []
|
||||
|
||||
class FakeMissingApp(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __call__(self, env, start_response):
|
||||
start_response('404 Not Found', [])
|
||||
return []
|
||||
|
||||
def start_response(*args):
|
||||
pass
|
||||
|
||||
class TestContainerQuotas(unittest.TestCase):
|
||||
|
||||
def test_not_handled(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'PUT'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_no_quotas(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': FakeCache({}),
|
||||
'CONTENT_LENGTH': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_exceed_bytes_quota(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
|
||||
'CONTENT_LENGTH': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 413)
|
||||
|
||||
def test_not_exceed_bytes_quota(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
|
||||
'CONTENT_LENGTH': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_exceed_counts_quota(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'}})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
|
||||
'CONTENT_LENGTH': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 413)
|
||||
|
||||
def test_not_exceed_counts_quota(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
cache = FakeCache({'count': 1, 'meta': {'quota-count': '2'}})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
|
||||
'CONTENT_LENGTH': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_invalid_quotas(self):
|
||||
req = Request.blank('/v1/a/c',
|
||||
environ={'REQUEST_METHOD': 'POST',
|
||||
'HTTP_X_CONTAINER_META_QUOTA_BYTES': 'abc'})
|
||||
res = req.get_response(
|
||||
container_quotas.ContainerQuotaMiddleware(FakeApp(), {}))
|
||||
self.assertEquals(res.status_int, 400)
|
||||
|
||||
req = Request.blank('/v1/a/c',
|
||||
environ={'REQUEST_METHOD': 'POST',
|
||||
'HTTP_X_CONTAINER_META_QUOTA_COUNT': 'abc'})
|
||||
res = req.get_response(
|
||||
container_quotas.ContainerQuotaMiddleware(FakeApp(), {}))
|
||||
self.assertEquals(res.status_int, 400)
|
||||
|
||||
def test_valid_quotas(self):
|
||||
req = Request.blank('/v1/a/c',
|
||||
environ={'REQUEST_METHOD': 'POST',
|
||||
'HTTP_X_CONTAINER_META_QUOTA_BYTES': '123'})
|
||||
res = req.get_response(
|
||||
container_quotas.ContainerQuotaMiddleware(FakeApp(), {}))
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
req = Request.blank('/v1/a/c',
|
||||
environ={'REQUEST_METHOD': 'POST',
|
||||
'HTTP_X_CONTAINER_META_QUOTA_COUNT': '123'})
|
||||
res = req.get_response(
|
||||
container_quotas.ContainerQuotaMiddleware(FakeApp(), {}))
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_delete_quotas(self):
|
||||
req = Request.blank('/v1/a/c',
|
||||
environ={'REQUEST_METHOD': 'POST',
|
||||
'HTTP_X_CONTAINER_META_QUOTA_BYTES': None})
|
||||
res = req.get_response(
|
||||
container_quotas.ContainerQuotaMiddleware(FakeApp(), {}))
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_missing_container(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeMissingApp(), {})
|
||||
cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
|
||||
'CONTENT_LENGTH': '100'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 404)
|
||||
|
||||
def test_auth_fail(self):
|
||||
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
|
||||
cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'},
|
||||
'write_acl': None})
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
|
||||
'CONTENT_LENGTH': '100',
|
||||
'swift.authorize': lambda *args: HTTPUnauthorized()})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 401)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user