Handle X-Copy-From header in account_quota mw
Content length of the copied object is checked before allowing the copy request according to the account quota set by Reseller. Fixes: bug #1200271 Change-Id: Ie4700f23466dd149ea5a497e6c72438cf52940fd
This commit is contained in:
Fabien Boucher
parent
21c322c35d
commit
fffc95c3cc
|
|
@ -48,8 +48,7 @@ post -m quota-bytes:
|
|||
|
||||
from swift.common.swob import HTTPForbidden, HTTPRequestEntityTooLarge, \
|
||||
HTTPBadRequest, wsgify
|
||||
|
||||
from swift.proxy.controllers.base import get_account_info
|
||||
from swift.proxy.controllers.base import get_account_info, get_object_info
|
||||
|
||||
|
||||
class AccountQuotaMiddleware(object):
|
||||
|
|
@ -68,7 +67,7 @@ class AccountQuotaMiddleware(object):
|
|||
return self.app
|
||||
|
||||
try:
|
||||
request.split_path(2, 4, rest_with_last=True)
|
||||
ver, acc, cont, obj = request.split_path(2, 4, rest_with_last=True)
|
||||
except ValueError:
|
||||
return self.app
|
||||
|
||||
|
|
@ -86,10 +85,22 @@ class AccountQuotaMiddleware(object):
|
|||
if new_quota is not None:
|
||||
return HTTPForbidden()
|
||||
|
||||
copy_from = request.headers.get('X-Copy-From')
|
||||
content_length = (request.content_length or 0)
|
||||
|
||||
if obj and copy_from:
|
||||
path = '/' + ver + '/' + acc + '/' + copy_from.lstrip('/')
|
||||
object_info = get_object_info(request.environ, self.app, path)
|
||||
if not object_info or not object_info['length']:
|
||||
content_length = 0
|
||||
else:
|
||||
content_length = int(object_info['length'])
|
||||
|
||||
account_info = get_account_info(request.environ, self.app)
|
||||
if not account_info or not account_info['bytes']:
|
||||
return self.app
|
||||
new_size = int(account_info['bytes']) + (request.content_length or 0)
|
||||
|
||||
new_size = int(account_info['bytes']) + content_length
|
||||
quota = int(account_info['meta'].get('quota-bytes', -1))
|
||||
|
||||
if 0 <= quota < new_size:
|
||||
|
|
|
|||
|
|
@ -152,6 +152,22 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
|
|||
}
|
||||
|
||||
|
||||
def headers_to_object_info(headers, status_int=HTTP_OK):
|
||||
"""
|
||||
Construct a cacheable dict of object info based on response headers.
|
||||
"""
|
||||
headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
|
||||
info = {'status': status_int,
|
||||
'length': headers.get('content-length'),
|
||||
'type': headers.get('content-type'),
|
||||
'etag': headers.get('etag'),
|
||||
'meta': dict((key[14:], value)
|
||||
for key, value in headers.iteritems()
|
||||
if key.startswith('x-object-meta-'))
|
||||
}
|
||||
return info
|
||||
|
||||
|
||||
def cors_validation(func):
|
||||
"""
|
||||
Decorator to check if the request is a CORS request and if so, if it's
|
||||
|
|
@ -213,6 +229,21 @@ def cors_validation(func):
|
|||
return wrapped
|
||||
|
||||
|
||||
def get_object_info(env, app, path=None, swift_source=None):
|
||||
"""
|
||||
Get the info structure for an object, based on env and app.
|
||||
This is useful to middlewares.
|
||||
Note: This call bypasses auth. Success does not imply that the
|
||||
request has authorization to the object.
|
||||
"""
|
||||
(version, account, container, obj) = \
|
||||
split_path(path or env['PATH_INFO'], 4, 4, True)
|
||||
info = _get_object_info(app, env, account, container, obj)
|
||||
if not info:
|
||||
info = headers_to_object_info({}, 0)
|
||||
return info
|
||||
|
||||
|
||||
def get_container_info(env, app, swift_source=None):
|
||||
"""
|
||||
Get the info structure for a container, based on env and app.
|
||||
|
|
@ -268,6 +299,19 @@ def _get_cache_key(account, container):
|
|||
return cache_key, env_key
|
||||
|
||||
|
||||
def get_object_env_key(account, container, obj):
|
||||
"""
|
||||
Get the keys for env (env_key) where info about object is cached
|
||||
:param account: The name of the account
|
||||
:param container: The name of the container
|
||||
:param obj: The name of the object
|
||||
:returns a string env_key
|
||||
"""
|
||||
env_key = 'swift.object/%s/%s/%s' % (account,
|
||||
container, obj)
|
||||
return env_key
|
||||
|
||||
|
||||
def _set_info_cache(app, env, account, container, resp):
|
||||
"""
|
||||
Cache info in both memcache and env.
|
||||
|
|
@ -315,6 +359,34 @@ def _set_info_cache(app, env, account, container, resp):
|
|||
env[env_key] = info
|
||||
|
||||
|
||||
def _set_object_info_cache(app, env, account, container, obj, resp):
|
||||
"""
|
||||
Cache object info env. Do not cache object informations in
|
||||
memcache. This is an intentional omission as it would lead
|
||||
to cache pressure. This is a per-request cache.
|
||||
|
||||
Caching is used to avoid unnecessary calls to object servers.
|
||||
This is a private function that is being called by GETorHEAD_base.
|
||||
Any attempt to GET or HEAD from the object server should use
|
||||
the GETorHEAD_base interface which would then set the cache.
|
||||
|
||||
:param app: the application object
|
||||
:param account: the unquoted account name
|
||||
:param container: the unquoted container name or None
|
||||
:param object: the unquoted object name or None
|
||||
:param resp: the response received or None if info cache should be cleared
|
||||
"""
|
||||
|
||||
env_key = get_object_env_key(account, container, obj)
|
||||
|
||||
if not resp:
|
||||
env.pop(env_key, None)
|
||||
return
|
||||
|
||||
info = headers_to_object_info(resp.headers, resp.status_int)
|
||||
env[env_key] = info
|
||||
|
||||
|
||||
def clear_info_cache(app, env, account, container=None):
|
||||
"""
|
||||
Clear the cached info in both memcache and env
|
||||
|
|
@ -406,6 +478,40 @@ def get_info(app, env, account, container=None, ret_not_found=False):
|
|||
return None
|
||||
|
||||
|
||||
def _get_object_info(app, env, account, container, obj):
|
||||
"""
|
||||
Get the info about object
|
||||
|
||||
Note: This call bypasses auth. Success does not imply that the
|
||||
request has authorization to the info.
|
||||
|
||||
:param app: the application object
|
||||
:param env: the environment used by the current request
|
||||
:param account: The unquoted name of the account
|
||||
:param container: The unquoted name of the container
|
||||
:param obj: The unquoted name of the object
|
||||
:returns: the cached info or None if cannot be retrieved
|
||||
"""
|
||||
env_key = get_object_env_key(account, container, obj)
|
||||
info = env.get(env_key)
|
||||
if info:
|
||||
return info
|
||||
# Not in cached, let's try the object servers
|
||||
path = '/v1/%s/%s/%s' % (account, container, obj)
|
||||
req = _prepare_pre_auth_info_request(env, path)
|
||||
# Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in
|
||||
# the environment under environ[env_key]. We will
|
||||
# pick the one from environ[env_key] and use it to set the caller env
|
||||
resp = req.get_response(app)
|
||||
try:
|
||||
info = resp.environ[env_key]
|
||||
env[env_key] = info
|
||||
return info
|
||||
except (KeyError, AttributeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class Controller(object):
|
||||
"""Base WSGI controller class for the proxy"""
|
||||
server_type = 'Base'
|
||||
|
|
@ -1017,6 +1123,12 @@ class Controller(object):
|
|||
_set_info_cache(self.app, req.environ, account, container, res)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
(account, container, obj) = split_path(req.path_info, 3, 3, True)
|
||||
_set_object_info_cache(self.app, req.environ, account,
|
||||
container, obj, res)
|
||||
except ValueError:
|
||||
pass
|
||||
return res
|
||||
|
||||
def is_origin_allowed(self, cors_info, origin):
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@ from swift.common.swob import Request
|
|||
from swift.common.middleware import account_quotas
|
||||
|
||||
from swift.proxy.controllers.base import _get_cache_key, \
|
||||
headers_to_account_info
|
||||
headers_to_account_info, get_object_env_key, \
|
||||
headers_to_object_info
|
||||
|
||||
|
||||
class FakeCache(object):
|
||||
|
|
@ -46,17 +47,22 @@ class FakeApp(object):
|
|||
self.headers = headers
|
||||
|
||||
def __call__(self, env, start_response):
|
||||
# Cache the account_info (same as a real application)
|
||||
cache_key, env_key = _get_cache_key('a', None)
|
||||
env[env_key] = headers_to_account_info(self.headers, 200)
|
||||
start_response('200 OK', self.headers)
|
||||
if env['REQUEST_METHOD'] == "HEAD" and \
|
||||
env['PATH_INFO'] == '/v1/a/c2/o2':
|
||||
env_key = get_object_env_key('a', 'c2', 'o2')
|
||||
env[env_key] = headers_to_object_info(self.headers, 200)
|
||||
start_response('200 OK', self.headers)
|
||||
elif env['REQUEST_METHOD'] == "HEAD" and \
|
||||
env['PATH_INFO'] == '/v1/a/c2/o3':
|
||||
start_response('404 Not Found', [])
|
||||
else:
|
||||
# Cache the account_info (same as a real application)
|
||||
cache_key, env_key = _get_cache_key('a', None)
|
||||
env[env_key] = headers_to_account_info(self.headers, 200)
|
||||
start_response('200 OK', self.headers)
|
||||
return []
|
||||
|
||||
|
||||
def start_response(*args):
|
||||
pass
|
||||
|
||||
|
||||
class TestAccountQuota(unittest.TestCase):
|
||||
|
||||
def test_unauthorized(self):
|
||||
|
|
@ -91,6 +97,43 @@ class TestAccountQuota(unittest.TestCase):
|
|||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 413)
|
||||
|
||||
def test_exceed_bytes_quota_copy_from(self):
|
||||
headers = [('x-account-bytes-used', '500'),
|
||||
('x-account-meta-quota-bytes', '1000'),
|
||||
('content-length', '1000')]
|
||||
app = account_quotas.AccountQuotaMiddleware(FakeApp(headers))
|
||||
cache = FakeCache(None)
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT',
|
||||
'swift.cache': cache},
|
||||
headers={'x-copy-from': '/c2/o2'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 413)
|
||||
|
||||
def test_not_exceed_bytes_quota_copy_from(self):
|
||||
headers = [('x-account-bytes-used', '0'),
|
||||
('x-account-meta-quota-bytes', '1000')]
|
||||
app = account_quotas.AccountQuotaMiddleware(FakeApp(headers))
|
||||
cache = FakeCache(None)
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT',
|
||||
'swift.cache': cache},
|
||||
headers={'x-copy-from': '/c2/o2'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_quota_copy_from_no_src(self):
|
||||
headers = [('x-account-bytes-used', '0'),
|
||||
('x-account-meta-quota-bytes', '1000')]
|
||||
app = account_quotas.AccountQuotaMiddleware(FakeApp(headers))
|
||||
cache = FakeCache(None)
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT',
|
||||
'swift.cache': cache},
|
||||
headers={'x-copy-from': '/c2/o3'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_exceed_bytes_quota_reseller(self):
|
||||
headers = [('x-account-bytes-used', '1000'),
|
||||
('x-account-meta-quota-bytes', '0')]
|
||||
|
|
@ -103,6 +146,19 @@ class TestAccountQuota(unittest.TestCase):
|
|||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_exceed_bytes_quota_reseller_copy_from(self):
|
||||
headers = [('x-account-bytes-used', '1000'),
|
||||
('x-account-meta-quota-bytes', '0')]
|
||||
app = account_quotas.AccountQuotaMiddleware(FakeApp(headers))
|
||||
cache = FakeCache(None)
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'PUT',
|
||||
'swift.cache': cache,
|
||||
'reseller_request': True},
|
||||
headers={'x-copy-from': 'c2/o2'})
|
||||
res = req.get_response(app)
|
||||
self.assertEquals(res.status_int, 200)
|
||||
|
||||
def test_bad_application_quota(self):
|
||||
headers = []
|
||||
app = account_quotas.AccountQuotaMiddleware(FakeBadApp(headers))
|
||||
|
|
|
|||
|
|
@ -16,9 +16,9 @@
|
|||
import unittest
|
||||
from mock import patch
|
||||
from swift.proxy.controllers.base import headers_to_container_info, \
|
||||
headers_to_account_info, get_container_info, get_container_memcache_key, \
|
||||
get_account_info, get_account_memcache_key, _get_cache_key, get_info, \
|
||||
Controller
|
||||
headers_to_account_info, headers_to_object_info, get_container_info, \
|
||||
get_container_memcache_key, get_account_info, get_account_memcache_key, \
|
||||
get_object_env_key, _get_cache_key, get_info, get_object_info, Controller
|
||||
from swift.common.swob import Request
|
||||
from swift.common.utils import split_path
|
||||
from test.unit import fake_http_connect, FakeRing, FakeMemcache
|
||||
|
|
@ -29,12 +29,18 @@ FakeResponse_status_int = 201
|
|||
|
||||
|
||||
class FakeResponse(object):
|
||||
def __init__(self, headers, env, account, container):
|
||||
def __init__(self, headers, env, account, container, obj):
|
||||
self.headers = headers
|
||||
self.status_int = FakeResponse_status_int
|
||||
self.environ = env
|
||||
cache_key, env_key = _get_cache_key(account, container)
|
||||
if container:
|
||||
if obj:
|
||||
env_key = get_object_env_key(account, container, obj)
|
||||
else:
|
||||
cache_key, env_key = _get_cache_key(account, container)
|
||||
|
||||
if account and container and obj:
|
||||
info = headers_to_object_info(headers, FakeResponse_status_int)
|
||||
elif account and container:
|
||||
info = headers_to_container_info(headers, FakeResponse_status_int)
|
||||
else:
|
||||
info = headers_to_account_info(headers, FakeResponse_status_int)
|
||||
|
|
@ -47,13 +53,18 @@ class FakeRequest(object):
|
|||
(version, account, container, obj) = split_path(path, 2, 4, True)
|
||||
self.account = account
|
||||
self.container = container
|
||||
stype = container and 'container' or 'account'
|
||||
self.headers = {'x-%s-object-count' % (stype): 1000,
|
||||
'x-%s-bytes-used' % (stype): 6666}
|
||||
self.obj = obj
|
||||
if obj:
|
||||
self.headers = {'content-length': 5555,
|
||||
'content-type': 'text/plain'}
|
||||
else:
|
||||
stype = container and 'container' or 'account'
|
||||
self.headers = {'x-%s-object-count' % (stype): 1000,
|
||||
'x-%s-bytes-used' % (stype): 6666}
|
||||
|
||||
def get_response(self, app):
|
||||
return FakeResponse(self.headers, self.environ, self.account,
|
||||
self.container)
|
||||
self.container, self.obj)
|
||||
|
||||
|
||||
class FakeCache(object):
|
||||
|
|
@ -73,6 +84,21 @@ class TestFuncs(unittest.TestCase):
|
|||
|
||||
def test_GETorHEAD_base(self):
|
||||
base = Controller(self.app)
|
||||
req = Request.blank('/a/c/o/with/slashes')
|
||||
with patch('swift.proxy.controllers.base.'
|
||||
'http_connect', fake_http_connect(200)):
|
||||
resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part',
|
||||
'/a/c/o/with/slashes')
|
||||
self.assertTrue('swift.object/a/c/o/with/slashes' in resp.environ)
|
||||
self.assertEqual(
|
||||
resp.environ['swift.object/a/c/o/with/slashes']['status'], 200)
|
||||
req = Request.blank('/a/c/o')
|
||||
with patch('swift.proxy.controllers.base.'
|
||||
'http_connect', fake_http_connect(200)):
|
||||
resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part',
|
||||
'/a/c/o')
|
||||
self.assertTrue('swift.object/a/c/o' in resp.environ)
|
||||
self.assertEqual(resp.environ['swift.object/a/c/o']['status'], 200)
|
||||
req = Request.blank('/a/c')
|
||||
with patch('swift.proxy.controllers.base.'
|
||||
'http_connect', fake_http_connect(200)):
|
||||
|
|
@ -273,6 +299,28 @@ class TestFuncs(unittest.TestCase):
|
|||
resp = get_account_info(req.environ, 'xxx')
|
||||
self.assertEquals(resp['bytes'], 3867)
|
||||
|
||||
def test_get_object_info_env(self):
|
||||
cached = {'status': 200,
|
||||
'length': 3333,
|
||||
'type': 'application/json',
|
||||
'meta': {}}
|
||||
env_key = get_object_env_key("account", "cont", "obj")
|
||||
req = Request.blank("/v1/account/cont/obj",
|
||||
environ={env_key: cached,
|
||||
'swift.cache': FakeCache({})})
|
||||
resp = get_object_info(req.environ, 'xxx')
|
||||
self.assertEquals(resp['length'], 3333)
|
||||
self.assertEquals(resp['type'], 'application/json')
|
||||
|
||||
def test_get_object_info_no_env(self):
|
||||
req = Request.blank("/v1/account/cont/obj",
|
||||
environ={'swift.cache': FakeCache({})})
|
||||
with patch('swift.proxy.controllers.base.'
|
||||
'_prepare_pre_auth_info_request', FakeRequest):
|
||||
resp = get_object_info(req.environ, 'xxx')
|
||||
self.assertEquals(resp['length'], 5555)
|
||||
self.assertEquals(resp['type'], 'text/plain')
|
||||
|
||||
def test_headers_to_container_info_missing(self):
|
||||
resp = headers_to_container_info({}, 404)
|
||||
self.assertEquals(resp['status'], 404)
|
||||
|
|
@ -331,3 +379,31 @@ class TestFuncs(unittest.TestCase):
|
|||
self.assertEquals(
|
||||
resp,
|
||||
headers_to_account_info(headers.items(), 200))
|
||||
|
||||
def test_headers_to_object_info_missing(self):
|
||||
resp = headers_to_object_info({}, 404)
|
||||
self.assertEquals(resp['status'], 404)
|
||||
self.assertEquals(resp['length'], None)
|
||||
self.assertEquals(resp['etag'], None)
|
||||
|
||||
def test_headers_to_object_info_meta(self):
|
||||
headers = {'X-Object-Meta-Whatevs': 14,
|
||||
'x-object-meta-somethingelse': 0}
|
||||
resp = headers_to_object_info(headers.items(), 200)
|
||||
self.assertEquals(len(resp['meta']), 2)
|
||||
self.assertEquals(resp['meta']['whatevs'], 14)
|
||||
self.assertEquals(resp['meta']['somethingelse'], 0)
|
||||
|
||||
def test_headers_to_object_info_values(self):
|
||||
headers = {
|
||||
'content-length': '1024',
|
||||
'content-type': 'application/json',
|
||||
}
|
||||
resp = headers_to_object_info(headers.items(), 200)
|
||||
self.assertEquals(resp['length'], '1024')
|
||||
self.assertEquals(resp['type'], 'application/json')
|
||||
|
||||
headers['x-unused-header'] = 'blahblahblah'
|
||||
self.assertEquals(
|
||||
resp,
|
||||
headers_to_object_info(headers.items(), 200))
|
||||
|
|
|
|||
Loading…
Reference in New Issue