From 0ec5eef91619a7e23b99d619c3f56ef0f7c6784b Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 29 Mar 2012 07:22:24 +0000 Subject: [PATCH] Update middleware to use common pre_authed funcs Update StaticWeb, FormPost, and TempURL to use common make_pre_authed_env and make_pre_authed_req functions. Change-Id: I32cc3cddff0d2aaeb6314578872707c1a37b4bce --- swift/common/middleware/formpost.py | 49 ++++++------ swift/common/middleware/staticweb.py | 51 ++++-------- swift/common/middleware/tempurl.py | 26 +++--- swift/common/wsgi.py | 83 +++++++++++++++----- test/unit/common/middleware/test_formpost.py | 2 +- 5 files changed, 115 insertions(+), 96 deletions(-) diff --git a/swift/common/middleware/formpost.py b/swift/common/middleware/formpost.py index dfef637730..19b2eb5bc9 100644 --- a/swift/common/middleware/formpost.py +++ b/swift/common/middleware/formpost.py @@ -111,6 +111,7 @@ from time import time from urllib import quote, unquote from swift.common.utils import get_logger, streq_const_time +from swift.common.wsgi import make_pre_authed_env #: The size of data to read from the form at any given time. @@ -296,6 +297,8 @@ class FormPost(object): self.conf = conf #: The logger to use with this middleware. self.logger = get_logger(conf, log_route='formpost') + #: The HTTP user agent to use with subrequests. + self.agent = '%(orig)s FormPost' def __call__(self, env, start_response): """ @@ -413,15 +416,9 @@ class FormPost(object): max_file_size = int(attributes.get('max_file_size') or 0) except ValueError: raise FormInvalid('max_file_size not an integer') - subenv = {'REQUEST_METHOD': 'PUT', - 'SCRIPT_NAME': '', - 'SERVER_NAME': env['SERVER_NAME'], - 'SERVER_PORT': env['SERVER_PORT'], - 'SERVER_PROTOCOL': env['SERVER_PROTOCOL'], - 'HTTP_TRANSFER_ENCODING': 'chunked', - 'wsgi.input': _CappedFileLikeObject(fp, max_file_size), - 'swift.cache': env['swift.cache']} - subenv['PATH_INFO'] = env['PATH_INFO'] + subenv = make_pre_authed_env(env, 'PUT', agent=self.agent) + subenv['HTTP_TRANSFER_ENCODING'] = 'chunked' + subenv['wsgi.input'] = _CappedFileLikeObject(fp, max_file_size) if subenv['PATH_INFO'][-1] != '/' and \ subenv['PATH_INFO'].count('/') < 4: subenv['PATH_INFO'] += '/' @@ -429,6 +426,8 @@ class FormPost(object): if 'content-type' in attributes: subenv['CONTENT_TYPE'] = \ attributes['content-type'] or 'application/octet-stream' + elif 'CONTENT_TYPE' in subenv: + del subenv['CONTENT_TYPE'] try: if int(attributes.get('expires') or 0) < time(): return '401 Unauthorized', 'form expired' @@ -445,15 +444,16 @@ class FormPost(object): if not streq_const_time(sig, (attributes.get('signature') or 'invalid')): return '401 Unauthorized', 'invalid signature' - subenv['swift.authorize'] = lambda req: None - subenv['swift.authorize_override'] = True - subenv['REMOTE_USER'] = '.wsgi.formpost' substatus = [None] def _start_response(status, headers, exc_info=None): substatus[0] = status - self.app(subenv, _start_response) + i = iter(self.app(subenv, _start_response)) + try: + i.next() + except StopIteration: + pass return substatus[0], '' def _get_key(self, env): @@ -474,19 +474,10 @@ class FormPost(object): if memcache: key = memcache.get('temp-url-key/%s' % account) if not key: - newenv = {'REQUEST_METHOD': 'HEAD', 'SCRIPT_NAME': '', - 'PATH_INFO': '/v1/' + account, 'CONTENT_LENGTH': '0', - 'SERVER_PROTOCOL': 'HTTP/1.0', - 'HTTP_USER_AGENT': 'FormPost', 'wsgi.version': (1, 0), - 'wsgi.url_scheme': 'http', 'wsgi.input': StringIO('')} - for name in ('SERVER_NAME', 'SERVER_PORT', 'wsgi.errors', - 'wsgi.multithread', 'wsgi.multiprocess', - 'wsgi.run_once', 'swift.cache', 'swift.trans_id'): - if name in env: - newenv[name] = env[name] - newenv['swift.authorize'] = lambda req: None - newenv['swift.authorize_override'] = True - newenv['REMOTE_USER'] = '.wsgi.formpost' + newenv = make_pre_authed_env(env, 'HEAD', '/v1/' + account, + self.agent) + newenv['CONTENT_LENGTH'] = '0' + newenv['wsgi.input'] = StringIO('') key = [None] def _start_response(status, response_headers, exc_info=None): @@ -494,7 +485,11 @@ class FormPost(object): if h.lower() == 'x-account-meta-temp-url-key': key[0] = v - self.app(newenv, _start_response) + i = iter(self.app(newenv, _start_response)) + try: + i.next() + except StopIteration: + pass key = key[0] if key and memcache: memcache.set('temp-url-key/%s' % account, key, timeout=60) diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index 01c8d11128..40f48c8950 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -123,7 +123,8 @@ from webob.exc import HTTPMovedPermanently, HTTPNotFound from swift.common.utils import cache_from_env, get_logger, human_readable, \ split_path, TRUE_VALUES -from swift.common.wsgi import WSGIContext +from swift.common.wsgi import make_pre_authed_env, make_pre_authed_request, \ + WSGIContext def quote(value, safe='/'): @@ -157,6 +158,7 @@ class _StaticWebContext(WSGIContext): self.logger = staticweb.logger self.access_logger = staticweb.access_logger self.log_headers = staticweb.log_headers + self.agent = '%(orig)s StaticWeb' # Results from the last call to self._get_container_info. self._index = self._error = self._listings = self._listings_css = None @@ -177,11 +179,10 @@ class _StaticWebContext(WSGIContext): save_response_status = self._response_status save_response_headers = self._response_headers save_response_exc_info = self._response_exc_info - tmp_env = self._get_escalated_env(env) - tmp_env['REQUEST_METHOD'] = 'GET' - tmp_env['PATH_INFO'] = '/%s/%s/%s/%s%s' % (self.version, self.account, - self.container, self._get_status_int(), self._error) - resp = self._app_call(tmp_env) + resp = self._app_call(make_pre_authed_env(env, 'GET', + '/%s/%s/%s/%s%s' % (self.version, self.account, self.container, + self._get_status_int(), self._error), + self.agent)) if self._get_status_int() // 100 == 2: start_response(save_response_status, self._response_headers, self._response_exc_info) @@ -190,21 +191,6 @@ class _StaticWebContext(WSGIContext): save_response_exc_info) return response - def _get_escalated_env(self, env): - """ - Returns a new fresh WSGI environment with escalated privileges to do - backend checks, listings, etc. that the remote user wouldn't be able to - accomplish directly. - """ - new_env = {'REQUEST_METHOD': 'GET', - 'HTTP_USER_AGENT': '%s StaticWeb' % env.get('HTTP_USER_AGENT')} - for name in ('eventlet.posthooks', 'swift.trans_id', 'REMOTE_USER', - 'SCRIPT_NAME', 'SERVER_NAME', 'SERVER_PORT', - 'SERVER_PROTOCOL', 'swift.cache'): - if name in env: - new_env[name] = env[name] - return new_env - def _get_container_info(self, env): """ Retrieves x-container-meta-web-index, x-container-meta-web-error, @@ -224,11 +210,9 @@ class _StaticWebContext(WSGIContext): (self._index, self._error, self._listings, self._listings_css) = cached_data return - tmp_env = self._get_escalated_env(env) - tmp_env['REQUEST_METHOD'] = 'HEAD' - req = Request.blank('/%s/%s/%s' % (self.version, self.account, - self.container), environ=tmp_env) - resp = req.get_response(self.app) + resp = make_pre_authed_request(env, 'HEAD', + '/%s/%s/%s' % (self.version, self.account, self.container), + agent=self.agent).get_response(self.app) if resp.status_int // 100 == 2: self._index = \ resp.headers.get('x-container-meta-web-index', '').strip() @@ -256,10 +240,9 @@ class _StaticWebContext(WSGIContext): if self._listings.lower() not in TRUE_VALUES: resp = HTTPNotFound()(env, self._start_response) return self._error_response(resp, env, start_response) - tmp_env = self._get_escalated_env(env) - tmp_env['REQUEST_METHOD'] = 'GET' - tmp_env['PATH_INFO'] = \ - '/%s/%s/%s' % (self.version, self.account, self.container) + tmp_env = make_pre_authed_env(env, 'GET', + '/%s/%s/%s' % (self.version, self.account, self.container), + self.agent) tmp_env['QUERY_STRING'] = 'delimiter=/&format=json' if prefix: tmp_env['QUERY_STRING'] += '&prefix=%s' % quote(prefix) @@ -431,10 +414,10 @@ class _StaticWebContext(WSGIContext): return resp if status_int == 404: if env['PATH_INFO'][-1] != '/': - tmp_env = self._get_escalated_env(env) - tmp_env['REQUEST_METHOD'] = 'GET' - tmp_env['PATH_INFO'] = '/%s/%s/%s' % (self.version, - self.account, self.container) + tmp_env = make_pre_authed_env(env, 'GET', + '/%s/%s/%s' % (self.version, self.account, + self.container), + self.agent) tmp_env['QUERY_STRING'] = 'limit=1&format=json&delimiter' \ '=/&limit=1&prefix=%s' % quote(self.obj + '/') resp = self._app_call(tmp_env) diff --git a/swift/common/middleware/tempurl.py b/swift/common/middleware/tempurl.py index 97419a14c3..a1467e818b 100644 --- a/swift/common/middleware/tempurl.py +++ b/swift/common/middleware/tempurl.py @@ -87,6 +87,7 @@ from urllib import quote, unquote from urlparse import parse_qs from swift.common.utils import get_logger +from swift.common.wsgi import make_pre_authed_env #: Default headers to remove from incoming requests. Simply a whitespace @@ -211,6 +212,8 @@ class TempURL(object): #: Lowercase, like `x-matches-remove-prefix-but-okay-*`. self.outgoing_allow_headers_startswith = \ [h[:-1] for h in headers if h[-1] == '*'] + #: HTTP user agent to use for subrequests. + self.agent = '%(orig)s TempURL' def __call__(self, env, start_response): """ @@ -321,19 +324,10 @@ class TempURL(object): if memcache: key = memcache.get('temp-url-key/%s' % account) if not key: - newenv = {'REQUEST_METHOD': 'HEAD', 'SCRIPT_NAME': '', - 'PATH_INFO': '/v1/' + account, 'CONTENT_LENGTH': '0', - 'SERVER_PROTOCOL': 'HTTP/1.0', - 'HTTP_USER_AGENT': 'TempURL', 'wsgi.version': (1, 0), - 'wsgi.url_scheme': 'http', 'wsgi.input': StringIO('')} - for name in ('SERVER_NAME', 'SERVER_PORT', 'wsgi.errors', - 'wsgi.multithread', 'wsgi.multiprocess', - 'wsgi.run_once', 'swift.cache', 'swift.trans_id'): - if name in env: - newenv[name] = env[name] - newenv['swift.authorize'] = lambda req: None - newenv['swift.authorize_override'] = True - newenv['REMOTE_USER'] = '.wsgi.tempurl' + newenv = make_pre_authed_env(env, 'HEAD', '/v1/' + account, + self.agent) + newenv['CONTENT_LENGTH'] = '0' + newenv['wsgi.input'] = StringIO('') key = [None] def _start_response(status, response_headers, exc_info=None): @@ -341,7 +335,11 @@ class TempURL(object): if h.lower() == 'x-account-meta-temp-url-key': key[0] = v - self.app(newenv, _start_response) + i = iter(self.app(newenv, _start_response)) + try: + i.next() + except StopIteration: + pass key = key[0] if key and memcache: memcache.set('temp-url-key/%s' % account, key, timeout=60) diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index f2834c1d2c..6df0b0dcfc 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -224,12 +224,12 @@ class WSGIContext(object): Ensures start_response has been called before returning. """ resp = iter(self.app(env, self._start_response)) - first_chunk = [] try: - first_chunk.append(resp.next()) + first_chunk = resp.next() except StopIteration: - pass - return chain(first_chunk, resp) + return iter([]) + else: # We got a first_chunk + return chain([first_chunk], resp) def _get_status_int(self): """ @@ -246,27 +246,29 @@ class WSGIContext(object): return None -def make_pre_authed_request(env, method, path, body=None, headers=None, - agent='Swift'): +def make_pre_authed_request(env, method=None, path=None, body=None, + headers=None, agent='Swift'): """ Makes a new webob.Request based on the current env but with the parameters specified. Note that this request will be preauthorized. - :param env: Current WSGI environment dictionary - :param method: HTTP method of new request - :param path: HTTP path of new request - :param body: HTTP body of new request; None by default - :param headers: Extra HTTP headers of new request; None by default - - :returns: webob.Request object - - (Stolen from Swauth: https://github.com/gholt/swauth) + :param env: The WSGI environment to base the new request on. + :param method: HTTP method of new request; default is from + the original env. + :param path: HTTP path of new request; default is from the + original env. + :param body: HTTP body of new request; empty by default. + :param headers: Extra HTTP headers of new request; None by + default. + :param agent: The HTTP user agent to use; default 'Swift'. You + can put %(orig)s in the agent to have it replaced + with the original env's HTTP_USER_AGENT, such as + '%(orig)s StaticWeb'. You also set agent to None to + use the original env's HTTP_USER_AGENT or '' to + have no HTTP_USER_AGENT. + :returns: Fresh webob.Request object. """ - newenv = {'REQUEST_METHOD': method, 'HTTP_USER_AGENT': agent} - for name in ('swift.cache', 'swift.trans_id'): - if name in env: - newenv[name] = env[name] - newenv['swift.authorize'] = lambda req: None + newenv = make_pre_authed_env(env, method, path, agent) if not headers: headers = {} if body: @@ -274,3 +276,44 @@ def make_pre_authed_request(env, method, path, body=None, headers=None, headers=headers) else: return Request.blank(path, environ=newenv, headers=headers) + + +def make_pre_authed_env(env, method=None, path=None, agent='Swift'): + """ + Returns a new fresh WSGI environment with escalated privileges to + do backend checks, listings, etc. that the remote user wouldn't + be able to accomplish directly. + + :param env: The WSGI environment to base the new environment on. + :param method: The new REQUEST_METHOD or None to use the + original. + :param path: The new PATH_INFO or None to use the original. + :param agent: The HTTP user agent to use; default 'Swift'. You + can put %(orig)s in the agent to have it replaced + with the original env's HTTP_USER_AGENT, such as + '%(orig)s StaticWeb'. You also set agent to None to + use the original env's HTTP_USER_AGENT or '' to + have no HTTP_USER_AGENT. + :returns: Fresh WSGI environment. + """ + newenv = {} + for name in ('eventlet.posthooks', 'HTTP_USER_AGENT', + 'PATH_INFO', 'REMOTE_USER', 'REQUEST_METHOD', + 'SCRIPT_NAME', 'SERVER_NAME', 'SERVER_PORT', + 'SERVER_PROTOCOL', 'swift.cache', 'swift.source', + 'swift.trans_id'): + if name in env: + newenv[name] = env[name] + if method: + newenv['REQUEST_METHOD'] = method + if path: + newenv['PATH_INFO'] = path + if agent: + newenv['HTTP_USER_AGENT'] = ( + agent % {'orig': env.get('HTTP_USER_AGENT', '')}).strip() + elif agent == '' and 'HTTP_USER_AGENT' in newenv: + del newenv['HTTP_USER_AGENT'] + newenv['swift.authorize'] = lambda req: None + newenv['swift.authorize_override'] = True + newenv['REMOTE_USER'] = '.wsgi.pre_authed' + return newenv diff --git a/test/unit/common/middleware/test_formpost.py b/test/unit/common/middleware/test_formpost.py index f94fc9c60b..cb77dcf871 100644 --- a/test/unit/common/middleware/test_formpost.py +++ b/test/unit/common/middleware/test_formpost.py @@ -74,7 +74,7 @@ class FakeApp(object): env['wsgi.input'] = StringIO(body) self.requests.append(Request.blank('', environ=env)) if env.get('swift.authorize_override') and \ - env.get('REMOTE_USER') != '.wsgi.formpost': + env.get('REMOTE_USER') != '.wsgi.pre_authed': raise Exception('Invalid REMOTE_USER %r with ' 'swift.authorize_override' % (env.get('REMOTE_USER'),)) if 'swift.authorize' in env: