Merge "add object-count quota for accounts in middleware"

This commit is contained in:
Zuul 2024-07-16 04:52:06 +00:00 committed by Gerrit Code Review
commit 7c12870068
2 changed files with 504 additions and 41 deletions

View File

@ -18,15 +18,29 @@
given account quota (in bytes) is exceeded while DELETE requests are still
allowed.
``account_quotas`` uses the ``x-account-meta-quota-bytes`` metadata entry to
store the overall account quota. Write requests to this metadata entry are
only permitted for resellers. There is no overall account quota limit if
``x-account-meta-quota-bytes`` is not set.
``account_quotas`` uses the following metadata entries to store the account
quota
Additionally, account quotas may be set for each storage policy, using metadata
of the form ``x-account-quota-bytes-policy-<policy name>``. Again, only
resellers may update these metadata, and there will be no limit for a
particular policy if the corresponding metadata is not set.
+---------------------------------------------+-------------------------------+
|Metadata | Use |
+=============================================+===============================+
| X-Account-Meta-Quota-Bytes | Maximum overall bytes stored |
| | in account across containers. |
+---------------------------------------------+-------------------------------+
| X-Account-Meta-Quota-Count | Maximum object count under |
| | account. |
+---------------------------------------------+-------------------------------+
Write requests to those metadata entries are only permitted for resellers.
There is no overall byte or object count limit set if the corresponding
metadata entries are not set.
Additionally, account quotas, of type quota-bytes or quota-count, may be set
for each storage policy, using metadata of the form ``x-account-<quota type>-\
policy-<policy name>``. Again, only resellers may update these metadata, and
there will be no limit for a particular policy if the corresponding metadata
is not set.
.. note::
Per-policy quotas need not sum to the overall account quota, and the sum of
@ -78,45 +92,50 @@ class AccountQuotaMiddleware(object):
def __init__(self, app, *args, **kwargs):
self.app = app
def validate_and_translate_quotas(self, request, quota_type):
new_quotas = {}
new_quotas[None] = request.headers.get(
'X-Account-Meta-%s' % quota_type)
if request.headers.get(
'X-Remove-Account-Meta-%s' % quota_type):
new_quotas[None] = 0 # X-Remove dominates if both are present
for policy in POLICIES:
tail = 'Account-%s-Policy-%s' % (quota_type, policy.name)
if request.headers.get('X-Remove-' + tail):
new_quotas[policy.idx] = 0
else:
quota = request.headers.pop('X-' + tail, None)
new_quotas[policy.idx] = quota
if request.environ.get('reseller_request') is True:
if any(quota and not quota.isdigit()
for quota in new_quotas.values()):
raise HTTPBadRequest()
for idx, quota in new_quotas.items():
if idx is None:
continue # For legacy reasons, it's in user meta
hdr = 'X-Account-Sysmeta-%s-Policy-%d' % (quota_type, idx)
request.headers[hdr] = quota
elif any(quota is not None for quota in new_quotas.values()):
# deny quota set for non-reseller
raise HTTPForbidden()
def handle_account(self, request):
if request.method in ("POST", "PUT"):
# account request, so we pay attention to the quotas
new_quotas = {}
new_quotas[None] = request.headers.get(
'X-Account-Meta-Quota-Bytes')
if request.headers.get(
'X-Remove-Account-Meta-Quota-Bytes'):
new_quotas[None] = 0 # X-Remove dominates if both are present
for policy in POLICIES:
tail = 'Account-Quota-Bytes-Policy-%s' % policy.name
if request.headers.get('X-Remove-' + tail):
new_quotas[policy.idx] = 0
else:
quota = request.headers.pop('X-' + tail, None)
new_quotas[policy.idx] = quota
if request.environ.get('reseller_request') is True:
if any(quota and not quota.isdigit()
for quota in new_quotas.values()):
return HTTPBadRequest()
for idx, quota in new_quotas.items():
if idx is None:
continue # For legacy reasons, it's in user meta
hdr = 'X-Account-Sysmeta-Quota-Bytes-Policy-%d' % idx
request.headers[hdr] = quota
elif any(quota is not None for quota in new_quotas.values()):
# deny quota set for non-reseller
return HTTPForbidden()
self.validate_and_translate_quotas(request, "Quota-Bytes")
self.validate_and_translate_quotas(request, "Quota-Count")
resp = request.get_response(self.app)
# Non-resellers can't update quotas, but they *can* see them
for policy in POLICIES:
infix = 'Quota-Bytes-Policy'
value = resp.headers.get('X-Account-Sysmeta-%s-%d' % (
infix, policy.idx))
if value:
resp.headers['X-Account-%s-%s' % (infix, policy.name)] = value
infixes = ('Quota-Bytes-Policy', 'Quota-Count-Policy')
for infix in infixes:
value = resp.headers.get('X-Account-Sysmeta-%s-%d' % (
infix, policy.idx))
if value:
resp.headers['X-Account-%s-%s' % (
infix, policy.name)] = value
return resp
@wsgify
@ -148,6 +167,8 @@ class AccountQuotaMiddleware(object):
swift_source='AQ')
if not account_info:
return self.app
# Check for quota byte violation
try:
quota = int(account_info['meta'].get('quota-bytes', -1))
except ValueError:
@ -168,11 +189,34 @@ class AccountQuotaMiddleware(object):
else:
return resp
# Check for quota count violation
try:
quota = int(account_info['meta'].get('quota-count', -1))
except ValueError:
quota = -1
if quota >= 0:
new_count = int(account_info['total_object_count']) + 1
if quota < new_count:
resp = HTTPRequestEntityTooLarge(body='Upload exceeds quota.')
if 'swift.authorize' in request.environ:
orig_authorize = request.environ['swift.authorize']
def reject_authorize(*args, **kwargs):
aresp = orig_authorize(*args, **kwargs)
if aresp:
return aresp
return resp
request.environ['swift.authorize'] = reject_authorize
else:
return resp
container_info = get_container_info(request.environ, self.app,
swift_source='AQ')
if not container_info:
return self.app
policy_idx = container_info['storage_policy']
# Check quota-byte per policy
sysmeta_key = 'quota-bytes-policy-%s' % policy_idx
try:
policy_quota = int(account_info['sysmeta'].get(sysmeta_key, -1))
@ -196,6 +240,30 @@ class AccountQuotaMiddleware(object):
else:
return resp
# Check quota-count per policy
sysmeta_key = 'quota-count-policy-%s' % policy_idx
try:
policy_quota = int(account_info['sysmeta'].get(sysmeta_key, -1))
except ValueError:
policy_quota = -1
if policy_quota >= 0:
policy_stats = account_info['storage_policies'].get(policy_idx, {})
new_size = int(policy_stats.get('object_count', 0)) + 1
if policy_quota < new_size:
resp = HTTPRequestEntityTooLarge(
body='Upload exceeds policy quota.')
if 'swift.authorize' in request.environ:
orig_authorize = request.environ['swift.authorize']
def reject_authorize(*args, **kwargs):
aresp = orig_authorize(*args, **kwargs)
if aresp:
return aresp
return resp
request.environ['swift.authorize'] = reject_authorize
else:
return resp
return self.app

View File

@ -498,6 +498,337 @@ class TestAccountQuota(unittest.TestCase):
res = req.get_response(app)
self.assertEqual(res.status_int, 503)
def test_obj_request_ignores_attempt_to_set_count_quotas(self):
# If you try to set X-Account-Meta-* on an object, it's ignored, so
# the quota middleware shouldn't complain about it even if we're not a
# reseller admin.
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
headers={'X-Account-Meta-Quota-Count': '99999'},
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_container_request_ignores_attempt_to_set_count_quotas(self):
# As with an object, if you try to set X-Account-Meta-* on a
# container, it's ignored.
self.app.register('PUT', '/v1/a/c', HTTPOk, {})
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a/c',
headers={'X-Account-Meta-Quota-Count': '99999'},
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_bogus_count_quota_is_ignored(self):
# This can happen if the metadata was set by a user prior to the
# activation of the account-quota middleware
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-bytes-used': '1000',
'x-account-meta-quota-count': 'pasty-plastogene'})
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_exceed_count_quota(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-object-count': '10',
'x-account-meta-quota-count': '10'})
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, b'Upload exceeds quota.')
def test_exceed_quota_count_not_authorized(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-meta-quota-count': '0'})
app = FakeAuthFilter(account_quotas.AccountQuotaMiddleware(self.app))
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o', method='PUT',
headers={'x-auth-token': 'bad-secret'},
environ={'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 403)
def test_exceed_count_quota_authorized(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-meta-quota-count': '0'})
app = FakeAuthFilter(account_quotas.AccountQuotaMiddleware(self.app))
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o', method='PUT',
headers={'x-auth-token': 'secret'},
environ={'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 413)
def test_under_quota_count_not_authorized(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-object-count': '0',
'x-account-meta-quota-count': '5'})
app = FakeAuthFilter(account_quotas.AccountQuotaMiddleware(self.app))
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o', method='PUT',
headers={'x-auth-token': 'bad-secret'},
environ={'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 403)
def test_under_quota_count_authorized(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-object-count': '0',
'x-account-meta-quota-count': '5'})
app = FakeAuthFilter(account_quotas.AccountQuotaMiddleware(self.app))
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o', method='PUT',
headers={'x-auth-token': 'secret'},
environ={'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_exceed_quota_count_on_empty_account_not_authorized(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-object-count': '0',
'x-account-meta-quota-count': '0'})
app = FakeAuthFilter(account_quotas.AccountQuotaMiddleware(self.app))
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o', method='PUT',
headers={'x-auth-token': 'bad-secret'},
environ={'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 403)
def test_exceed_quota_count_authorized(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-object-count': '5',
'x-account-meta-quota-count': '5'})
app = FakeAuthFilter(account_quotas.AccountQuotaMiddleware(self.app))
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o', method='PUT',
headers={'x-auth-token': 'secret'},
environ={'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, b'Upload exceeds quota.')
def test_over_quota_count_container_create_still_works(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-object-count': '6',
'x-account-meta-quota-count': '5'})
self.app.register('PUT', '/v1/a/new_container', HTTPOk, {})
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a/new_container',
environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_CONTAINER_META_BERT': 'ernie',
'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_over_quota_count_container_post_still_works(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-quota-count': '6',
'x-account-meta-quota-count': '5'})
self.app.register('POST', '/v1/a/new_container', HTTPOk, {})
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a/new_container',
environ={'REQUEST_METHOD': 'POST',
'HTTP_X_CONTAINER_META_BERT': 'ernie',
'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_over_count_quota_obj_post_still_works(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-object-count': '101',
'x-account-meta-quota-count': '100'})
self.app.register('POST', '/v1/a/c/o', HTTPOk, {})
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'POST',
'HTTP_X_OBJECT_META_BERT': 'ernie',
'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_exceed_count_quota_reseller(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-object-count': '1000',
'x-account-meta-quota-count': '0'})
self.app.register('PUT', '/v1/a', HTTPOk, {})
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache,
'reseller_request': True})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_exceed_count_quota_reseller_copy_from(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-object-count': '10',
'x-account-meta-quota-count': '10'})
self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {
'content-length': '1000'}, b'a' * 1000)
app = copy.filter_factory({})(
account_quotas.AccountQuotaMiddleware(self.app))
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache,
'reseller_request': True},
headers={'x-copy-from': 'c2/o2'})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_exceed_count_quota_reseller_copy_verb(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-object-count': '99',
'x-account-meta-quota-count': '100'})
self.app.register('GET', '/v1/a/c2/o2', HTTPOk, {
'content-length': '1000'}, b'a' * 1000)
app = copy.filter_factory({})(
account_quotas.AccountQuotaMiddleware(self.app))
cache = FakeCache(None)
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache,
'reseller_request': True},
headers={'Destination': 'c/o'})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_not_exceed_count_quota(self):
self.app.register('HEAD', '/v1/a', HTTPOk, {
'x-account-object-count': '10',
'x-account-meta-quota-count': '20'})
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_invalid_count_quotas_on_object(self):
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a',
environ={'REQUEST_METHOD': 'POST',
'swift.cache': cache,
'HTTP_X_ACCOUNT_META_QUOTA_COUNT': 'abc',
'reseller_request': True})
res = req.get_response(app)
self.assertEqual(res.status_int, 400)
self.assertEqual(self.app.calls, [])
def test_valid_count_quotas_admin(self):
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a',
environ={'REQUEST_METHOD': 'POST',
'swift.cache': cache,
'HTTP_X_ACCOUNT_META_QUOTA_COUNT': '100'})
res = req.get_response(app)
self.assertEqual(res.status_int, 403)
self.assertEqual(self.app.calls, [])
@patch_policies
def test_valid_policy_count_quota_admin(self):
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a', environ={
'REQUEST_METHOD': 'POST',
'swift.cache': cache,
'HTTP_X_ACCOUNT_QUOTA_COUNT_POLICY_UNU': '100'})
res = req.get_response(app)
self.assertEqual(res.status_int, 403)
self.assertEqual(self.app.calls, [])
def test_valid_count_quota_reseller(self):
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a',
environ={'REQUEST_METHOD': 'POST',
'swift.cache': cache,
'HTTP_X_ACCOUNT_META_QUOTA_COUNT': '100',
'reseller_request': True})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
self.assertEqual(self.app.calls_with_headers, [
('POST', '/v1/a', {'Host': 'localhost:80',
'X-Account-Meta-Quota-Count': '100'})])
@patch_policies
def test_valid_policy_count_quota_reseller(self):
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a', environ={
'REQUEST_METHOD': 'POST',
'swift.cache': cache,
'HTTP_X_ACCOUNT_QUOTA_COUNT_POLICY_NULO': '100',
'reseller_request': True})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
self.assertEqual(self.app.calls_with_headers, [
('POST', '/v1/a', {
'Host': 'localhost:80',
'X-Account-Sysmeta-Quota-Count-Policy-0': '100'})])
def test_delete_count_quotas(self):
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a',
environ={'REQUEST_METHOD': 'POST',
'swift.cache': cache,
'HTTP_X_ACCOUNT_META_QUOTA_COUNT': ''})
res = req.get_response(app)
self.assertEqual(res.status_int, 403)
def test_delete_count_quotas_with_remove_header(self):
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a', environ={
'REQUEST_METHOD': 'POST',
'swift.cache': cache,
'HTTP_X_REMOVE_ACCOUNT_META_QUOTA_COUNT': 'True'})
res = req.get_response(app)
self.assertEqual(res.status_int, 403)
def test_delete_count_quotas_reseller(self):
app = account_quotas.AccountQuotaMiddleware(self.app)
req = Request.blank('/v1/a',
environ={'REQUEST_METHOD': 'POST',
'HTTP_X_ACCOUNT_META_QUOTA_COUNT': '',
'reseller_request': True})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
def test_delete_count_quotas_with_remove_header_reseller(self):
app = account_quotas.AccountQuotaMiddleware(self.app)
cache = FakeCache(None)
req = Request.blank('/v1/a', environ={
'REQUEST_METHOD': 'POST',
'swift.cache': cache,
'HTTP_X_REMOVE_ACCOUNT_META_QUOTA_COUNT': 'True',
'reseller_request': True})
res = req.get_response(app)
self.assertEqual(res.status_int, 200)
class AccountQuotaCopyingTestCases(unittest.TestCase):
@ -576,6 +907,70 @@ class AccountQuotaCopyingTestCases(unittest.TestCase):
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 412)
def test_exceed_bytes_count_quota_copy_from(self):
self.headers[:] = [('x-account-object-count', '5'),
('x-account-meta-quota-count', '5')]
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, b'Upload exceeds quota.')
def test_exceed_bytes_count_quota_copy_verb(self):
self.headers[:] = [('x-account-object-count', '5'),
('x-account-meta-quota-count', '5')]
cache = FakeCache(None)
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 413)
self.assertEqual(res.body, b'Upload exceeds quota.')
def test_not_exceed_bytes_count_quota_copy_from(self):
self.app.register('PUT', '/v1/a/c/o', HTTPOk, {})
self.headers[:] = [('x-account-object-count', '5'),
('x-account-meta-quota-count', '6')]
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': '/c2/o2'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_not_exceed_bytes_count_quota_copy_verb(self):
self.app.register('PUT', '/v1/a/c/o', HTTPOk, {})
self.headers[:] = [('x-account-object-count', '5'),
('x-account-meta-quota-count', '6')]
cache = FakeCache(None)
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.cache': cache},
headers={'Destination': '/c/o'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 200)
def test_count_quota_copy_from_bad_src(self):
self.headers[:] = [('x-account-object-count', '0'),
('x-account-meta-quota-count', '1')]
cache = FakeCache(None)
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.cache': cache},
headers={'x-copy-from': 'bad_path'})
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 412)
self.headers[:] = [('x-account-object-count', '1'),
('x-account-meta-quota-count', '0')]
res = req.get_response(self.copy_filter)
self.assertEqual(res.status_int, 412)
if __name__ == '__main__':
unittest.main()