From 489dd5ff5db0590f5232e5c309bf50ba9481f804 Mon Sep 17 00:00:00 2001 From: Richard Hawkins Date: Mon, 9 Feb 2015 17:51:01 -0600 Subject: [PATCH] Add support for container TempURL Keys Change-Id: Ic22b0b84b657e6cac7e0062fa410eefb09bc0f4d Co-Authored-By: Christian Schwede --- etc/proxy-server.conf-sample | 2 +- swift/common/middleware/formpost.py | 25 +++++++-- swift/common/middleware/tempurl.py | 29 +++++++---- swift/proxy/server.py | 1 + test/unit/common/middleware/test_formpost.py | 55 ++++++++++++++++++++ test/unit/common/middleware/test_tempurl.py | 37 +++++++++++-- 6 files changed, 128 insertions(+), 21 deletions(-) diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index c825ec3b30..1d38e7c7e2 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -203,7 +203,7 @@ use = egg:swift#proxy # These are the headers whose values will only be shown to swift_owners. The # exact definition of a swift_owner is up to the auth system in use, but # usually indicates administrative responsibilities. -# swift_owner_headers = x-container-read, x-container-write, x-container-sync-key, x-container-sync-to, x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, x-account-access-control +# swift_owner_headers = x-container-read, x-container-write, x-container-sync-key, x-container-sync-to, x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, x-container-meta-temp-url-key, x-container-meta-temp-url-key-2, x-account-access-control [filter:tempauth] use = egg:swift#tempauth diff --git a/swift/common/middleware/formpost.py b/swift/common/middleware/formpost.py index 1a79a3e280..7132b342a5 100644 --- a/swift/common/middleware/formpost.py +++ b/swift/common/middleware/formpost.py @@ -90,8 +90,9 @@ sample code for computing the signature:: max_file_size, max_file_count, expires) signature = hmac.new(key, hmac_body, sha1).hexdigest() -The key is the value of either the X-Account-Meta-Temp-URL-Key or the -X-Account-Meta-Temp-Url-Key-2 header on the account. +The key is the value of either the account (X-Account-Meta-Temp-URL-Key, +X-Account-Meta-Temp-Url-Key-2) or the container +(X-Container-Meta-Temp-URL-Key, X-Container-Meta-Temp-Url-Key-2) TempURL keys. Be certain to use the full path, from the /v1/ onward. Note that x_delete_at and x_delete_after are not used in signature generation @@ -123,7 +124,7 @@ from swift.common.utils import streq_const_time, register_swift_info, \ parse_content_disposition, iter_multipart_mime_documents from swift.common.wsgi import make_pre_authed_env from swift.common.swob import HTTPUnauthorized -from swift.proxy.controllers.base import get_account_info +from swift.proxy.controllers.base import get_account_info, get_container_info #: The size of data to read from the form at any given time. @@ -393,7 +394,13 @@ class FormPost(object): def _get_keys(self, env): """ - Fetch the tempurl keys for the account. Also validate that the request + Returns the X-[Account|Container]-Meta-Temp-URL-Key[-2] header values + for the account or container, or an empty list if none are set. + + Returns 0-4 elements depending on how many keys are set in the + account's or container's metadata. + + Also validate that the request path indicates a valid container; if not, no keys will be returned. :param env: The WSGI environment for the request. @@ -405,12 +412,20 @@ class FormPost(object): return [] account_info = get_account_info(env, self.app, swift_source='FP') - return get_tempurl_keys_from_metadata(account_info['meta']) + account_keys = get_tempurl_keys_from_metadata(account_info['meta']) + + container_info = get_container_info(env, self.app, swift_source='FP') + container_keys = get_tempurl_keys_from_metadata( + container_info.get('meta', [])) + + return account_keys + container_keys def filter_factory(global_conf, **local_conf): """Returns the WSGI filter for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) + register_swift_info('formpost') + return lambda app: FormPost(app, conf) diff --git a/swift/common/middleware/tempurl.py b/swift/common/middleware/tempurl.py index c2381b3183..3dd1448583 100644 --- a/swift/common/middleware/tempurl.py +++ b/swift/common/middleware/tempurl.py @@ -81,10 +81,13 @@ Using this in combination with browser form post translation middleware could also allow direct-from-browser uploads to specific locations in Swift. -TempURL supports up to two keys, specified by X-Account-Meta-Temp-URL-Key and -X-Account-Meta-Temp-URL-Key-2. Signatures are checked against both keys, if -present. This is to allow for key rotation without invalidating all existing -temporary URLs. +TempURL supports both account and container level keys. Each allows up to two +keys to be set, allowing key rotation without invalidating all existing +temporary URLs. Account keys are specified by X-Account-Meta-Temp-URL-Key and +X-Account-Meta-Temp-URL-Key-2, while container keys are specified by +X-Container-Meta-Temp-URL-Key and X-Container-Meta-Temp-URL-Key-2. +Signatures are checked against account and container keys, if +present. With GET TempURLs, a Content-Disposition header will be set on the response so that browsers will interpret this as a file attachment to @@ -118,7 +121,7 @@ from time import time from urllib import urlencode from urlparse import parse_qs -from swift.proxy.controllers.base import get_account_info +from swift.proxy.controllers.base import get_account_info, get_container_info from swift.common.swob import HeaderKeyDict, HTTPUnauthorized from swift.common.utils import split_path, get_valid_utf8_str, \ register_swift_info, get_hmac, streq_const_time, quote @@ -409,11 +412,11 @@ class TempURL(object): def _get_keys(self, env, account): """ - Returns the X-Account-Meta-Temp-URL-Key[-2] header values for the - account, or an empty list if none is set. + Returns the X-[Account|Container]-Meta-Temp-URL-Key[-2] header values + for the account or container, or an empty list if none are set. - Returns 0, 1, or 2 elements depending on how many keys are set - in the account's metadata. + Returns 0-4 elements depending on how many keys are set in the + account's or container's metadata. :param env: The WSGI environment for the request. :param account: Account str. @@ -421,7 +424,13 @@ class TempURL(object): X-Account-Meta-Temp-URL-Key-2 str value if set] """ account_info = get_account_info(env, self.app, swift_source='TU') - return get_tempurl_keys_from_metadata(account_info['meta']) + account_keys = get_tempurl_keys_from_metadata(account_info['meta']) + + container_info = get_container_info(env, self.app, swift_source='TU') + container_keys = get_tempurl_keys_from_metadata( + container_info.get('meta', [])) + + return account_keys + container_keys def _get_hmacs(self, env, expires, keys, request_method=None): """ diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 7920fe42db..28d41df554 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -186,6 +186,7 @@ class Application(object): 'x-container-read, x-container-write, ' 'x-container-sync-key, x-container-sync-to, ' 'x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, ' + 'x-container-meta-temp-url-key, x-container-meta-temp-url-key-2, ' 'x-account-access-control') self.swift_owner_headers = [ name.strip().title() diff --git a/test/unit/common/middleware/test_formpost.py b/test/unit/common/middleware/test_formpost.py index 9ba713c2c2..c71eb7cc83 100644 --- a/test/unit/common/middleware/test_formpost.py +++ b/test/unit/common/middleware/test_formpost.py @@ -345,6 +345,7 @@ class TestFormPost(unittest.TestCase): 'SERVER_PROTOCOL': 'HTTP/1.0', 'swift.account/AUTH_test': self._fake_cache_env( 'AUTH_test', [key]), + 'swift.container/AUTH_test/container': {'meta': {}}, 'wsgi.errors': wsgi_errors, 'wsgi.input': wsgi_input, 'wsgi.multiprocess': False, @@ -457,6 +458,7 @@ class TestFormPost(unittest.TestCase): 'SERVER_PROTOCOL': 'HTTP/1.0', 'swift.account/AUTH_test': self._fake_cache_env( 'AUTH_test', [key]), + 'swift.container/AUTH_test/container': {'meta': {}}, 'wsgi.errors': wsgi_errors, 'wsgi.input': wsgi_input, 'wsgi.multiprocess': False, @@ -572,6 +574,7 @@ class TestFormPost(unittest.TestCase): 'SERVER_PROTOCOL': 'HTTP/1.0', 'swift.account/AUTH_test': self._fake_cache_env( 'AUTH_test', [key]), + 'swift.container/AUTH_test/container': {'meta': {}}, 'wsgi.errors': wsgi_errors, 'wsgi.input': wsgi_input, 'wsgi.multiprocess': False, @@ -683,6 +686,7 @@ class TestFormPost(unittest.TestCase): 'SERVER_PROTOCOL': 'HTTP/1.0', 'swift.account/AUTH_test': self._fake_cache_env( 'AUTH_test', [key]), + 'swift.container/AUTH_test/container': {'meta': {}}, 'wsgi.errors': wsgi_errors, 'wsgi.input': wsgi_input, 'wsgi.multiprocess': False, @@ -728,6 +732,7 @@ class TestFormPost(unittest.TestCase): env['wsgi.input'] = StringIO('XX' + '\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -763,6 +768,7 @@ class TestFormPost(unittest.TestCase): env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -793,6 +799,7 @@ class TestFormPost(unittest.TestCase): env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -833,6 +840,7 @@ class TestFormPost(unittest.TestCase): env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp( iter([('201 Created', {}, ''), ('201 Created', {}, '')]), @@ -867,6 +875,7 @@ class TestFormPost(unittest.TestCase): env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp(iter([('404 Not Found', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -949,6 +958,7 @@ class TestFormPost(unittest.TestCase): ])) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1016,6 +1026,7 @@ class TestFormPost(unittest.TestCase): ])) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1055,6 +1066,7 @@ class TestFormPost(unittest.TestCase): env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1075,6 +1087,7 @@ class TestFormPost(unittest.TestCase): env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} env['HTTP_ORIGIN'] = 'http://localhost:5000' self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', @@ -1103,6 +1116,7 @@ class TestFormPost(unittest.TestCase): # Stick it in X-Account-Meta-Temp-URL-Key-2 and make sure we get it env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', ['bert', key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1120,6 +1134,42 @@ class TestFormPost(unittest.TestCase): 'http://redirect?status=201&message=', dict(headers[0]).get('Location')) + def test_formpost_with_multiple_container_keys(self): + first_key = 'ernie' + second_key = 'bert' + keys = [first_key, second_key] + + meta = {} + for idx, key in enumerate(keys): + meta_name = 'temp-url-key' + ("-%d" % (idx + 1) if idx else "") + if key: + meta[meta_name] = key + + for key in keys: + sig, env, body = self._make_sig_env_body( + '/v1/AUTH_test/container', 'http://redirect', 1024, 10, + int(time() + 86400), key) + env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['swift.account/AUTH_test'] = self._fake_cache_env('AUTH_test') + # Stick it in X-Container-Meta-Temp-URL-Key-2 and ensure we get it + env['swift.container/AUTH_test/container'] = {'meta': meta} + self.app = FakeApp(iter([('201 Created', {}, ''), + ('201 Created', {}, '')])) + self.auth = tempauth.filter_factory({})(self.app) + self.formpost = formpost.filter_factory({})(self.auth) + + status = [None] + headers = [None] + + def start_response(s, h, e=None): + status[0] = s + headers[0] = h + body = ''.join(self.formpost(env, start_response)) + self.assertEqual('303 See Other', status[0]) + self.assertEqual( + 'http://redirect?status=201&message=', + dict(headers[0]).get('Location')) + def test_redirect(self): key = 'abc' sig, env, body = self._make_sig_env_body( @@ -1128,6 +1178,7 @@ class TestFormPost(unittest.TestCase): env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1165,6 +1216,7 @@ class TestFormPost(unittest.TestCase): env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1202,6 +1254,7 @@ class TestFormPost(unittest.TestCase): env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1550,6 +1603,7 @@ class TestFormPost(unittest.TestCase): env['wsgi.input'] = StringIO('\r\n'.join(x_delete_body_part + body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1625,6 +1679,7 @@ class TestFormPost(unittest.TestCase): env['wsgi.input'] = StringIO('\r\n'.join(x_delete_body_part + body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) + env['swift.container/AUTH_test/container'] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) diff --git a/test/unit/common/middleware/test_tempurl.py b/test/unit/common/middleware/test_tempurl.py index 0581077094..cc04aa3ea5 100644 --- a/test/unit/common/middleware/test_tempurl.py +++ b/test/unit/common/middleware/test_tempurl.py @@ -97,6 +97,9 @@ class TestTempURL(unittest.TestCase): 'bytes': '0', 'meta': meta} + container_cache_key = 'swift.container/' + account + '/c' + environ.setdefault(container_cache_key, {'meta': {}}) + def test_passthrough(self): resp = self._make_request('/v1/a/c/o').get_response(self.tempurl) self.assertEquals(resp.status_int, 401) @@ -109,11 +112,12 @@ class TestTempURL(unittest.TestCase): environ={'REQUEST_METHOD': 'OPTIONS'}).get_response(self.tempurl) self.assertEquals(resp.status_int, 200) - def assert_valid_sig(self, expires, path, keys, sig): - req = self._make_request( - path, keys=keys, - environ={'QUERY_STRING': - 'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)}) + def assert_valid_sig(self, expires, path, keys, sig, environ=None): + if not environ: + environ = {} + environ['QUERY_STRING'] = 'temp_url_sig=%s&temp_url_expires=%s' % ( + sig, expires) + req = self._make_request(path, keys=keys, environ=environ) self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')])) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 200) @@ -143,6 +147,29 @@ class TestTempURL(unittest.TestCase): for sig in (sig1, sig2): self.assert_valid_sig(expires, path, [key1, key2], sig) + def test_get_valid_container_keys(self): + environ = {} + # Add two static container keys + container_keys = ['me', 'other'] + meta = {} + for idx, key in enumerate(container_keys): + meta_name = 'Temp-URL-key' + (("-%d" % (idx + 1) if idx else "")) + if key: + meta[meta_name] = key + environ['swift.container/a/c'] = {'meta': meta} + + method = 'GET' + expires = int(time() + 86400) + path = '/v1/a/c/o' + key1 = 'me' + key2 = 'other' + hmac_body = '%s\n%s\n%s' % (method, expires, path) + sig1 = hmac.new(key1, hmac_body, sha1).hexdigest() + sig2 = hmac.new(key2, hmac_body, sha1).hexdigest() + account_keys = [] + for sig in (sig1, sig2): + self.assert_valid_sig(expires, path, account_keys, sig, environ) + def test_get_valid_with_filename(self): method = 'GET' expires = int(time() + 86400)