openstack
/
swift
Code Issues Proposed changes

Merge "Update middleware to use common pre_authed funcs"

This commit is contained in:
Jenkins 2012-03-30 19:33:39 +00:00 committed by Gerrit Code Review
parent d46411d4fd 0ec5eef916
commit a9b6355d15
5 changed files with 115 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
swift/common/middleware/formpost.py
View File

@ -111,6 +111,7 @@ from time import time
from urllib import quote, unquote
from swift.common.utils import get_logger, streq_const_time
from swift.common.wsgi import make_pre_authed_env
#: The size of data to read from the form at any given time.
@ -296,6 +297,8 @@ class FormPost(object):
self.conf = conf
#: The logger to use with this middleware.
self.logger = get_logger(conf, log_route='formpost')
#: The HTTP user agent to use with subrequests.
self.agent = '%(orig)s FormPost'
def __call__(self, env, start_response):
"""
@ -413,15 +416,9 @@ class FormPost(object):
max_file_size = int(attributes.get('max_file_size') or 0)
except ValueError:
raise FormInvalid('max_file_size not an integer')
subenv = {'REQUEST_METHOD': 'PUT',
'SCRIPT_NAME': '',
'SERVER_NAME': env['SERVER_NAME'],
'SERVER_PORT': env['SERVER_PORT'],
'SERVER_PROTOCOL': env['SERVER_PROTOCOL'],
'HTTP_TRANSFER_ENCODING': 'chunked',
'wsgi.input': _CappedFileLikeObject(fp, max_file_size),
'swift.cache': env['swift.cache']}
subenv['PATH_INFO'] = env['PATH_INFO']
subenv = make_pre_authed_env(env, 'PUT', agent=self.agent)
subenv['HTTP_TRANSFER_ENCODING'] = 'chunked'
subenv['wsgi.input'] = _CappedFileLikeObject(fp, max_file_size)
if subenv['PATH_INFO'][-1] != '/' and \
subenv['PATH_INFO'].count('/') < 4:
subenv['PATH_INFO'] += '/'
@ -429,6 +426,8 @@ class FormPost(object):
if 'content-type' in attributes:
subenv['CONTENT_TYPE'] = \
attributes['content-type'] or 'application/octet-stream'
elif 'CONTENT_TYPE' in subenv:
del subenv['CONTENT_TYPE']
try:
if int(attributes.get('expires') or 0) < time():
return '401 Unauthorized', 'form expired'
@ -445,15 +444,16 @@ class FormPost(object):
if not streq_const_time(sig, (attributes.get('signature') or
'invalid')):
return '401 Unauthorized', 'invalid signature'
subenv['swift.authorize'] = lambda req: None
subenv['swift.authorize_override'] = True
subenv['REMOTE_USER'] = '.wsgi.formpost'
substatus = [None]
def _start_response(status, headers, exc_info=None):
substatus[0] = status
self.app(subenv, _start_response)
i = iter(self.app(subenv, _start_response))
try:
i.next()
except StopIteration:
pass
return substatus[0], ''
def _get_key(self, env):
@ -474,19 +474,10 @@ class FormPost(object):
if memcache:
key = memcache.get('temp-url-key/%s' % account)
if not key:
newenv = {'REQUEST_METHOD': 'HEAD', 'SCRIPT_NAME': '',
'PATH_INFO': '/v1/' + account, 'CONTENT_LENGTH': '0',
'SERVER_PROTOCOL': 'HTTP/1.0',
'HTTP_USER_AGENT': 'FormPost', 'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http', 'wsgi.input': StringIO('')}
for name in ('SERVER_NAME', 'SERVER_PORT', 'wsgi.errors',
'wsgi.multithread', 'wsgi.multiprocess',
'wsgi.run_once', 'swift.cache', 'swift.trans_id'):
if name in env:
newenv[name] = env[name]
newenv['swift.authorize'] = lambda req: None
newenv['swift.authorize_override'] = True
newenv['REMOTE_USER'] = '.wsgi.formpost'
newenv = make_pre_authed_env(env, 'HEAD', '/v1/' + account,
self.agent)
newenv['CONTENT_LENGTH'] = '0'
newenv['wsgi.input'] = StringIO('')
key = [None]
def _start_response(status, response_headers, exc_info=None):
@ -494,7 +485,11 @@ class FormPost(object):
if h.lower() == 'x-account-meta-temp-url-key':
key[0] = v
self.app(newenv, _start_response)
i = iter(self.app(newenv, _start_response))
try:
i.next()
except StopIteration:
pass
key = key[0]
if key and memcache:
memcache.set('temp-url-key/%s' % account, key, timeout=60)

51
swift/common/middleware/staticweb.py
View File

@ -123,7 +123,8 @@ from webob.exc import HTTPMovedPermanently, HTTPNotFound
from swift.common.utils import cache_from_env, get_logger, human_readable, \
split_path, TRUE_VALUES
from swift.common.wsgi import WSGIContext
from swift.common.wsgi import make_pre_authed_env, make_pre_authed_request, \
WSGIContext
def quote(value, safe='/'):
@ -157,6 +158,7 @@ class _StaticWebContext(WSGIContext):
self.logger = staticweb.logger
self.access_logger = staticweb.access_logger
self.log_headers = staticweb.log_headers
self.agent = '%(orig)s StaticWeb'
# Results from the last call to self._get_container_info.
self._index = self._error = self._listings = self._listings_css = None
@ -177,11 +179,10 @@ class _StaticWebContext(WSGIContext):
save_response_status = self._response_status
save_response_headers = self._response_headers
save_response_exc_info = self._response_exc_info
tmp_env = self._get_escalated_env(env)
tmp_env['REQUEST_METHOD'] = 'GET'
tmp_env['PATH_INFO'] = '/%s/%s/%s/%s%s' % (self.version, self.account,
self.container, self._get_status_int(), self._error)
resp = self._app_call(tmp_env)
resp = self._app_call(make_pre_authed_env(env, 'GET',
'/%s/%s/%s/%s%s' % (self.version, self.account, self.container,
self._get_status_int(), self._error),
self.agent))
if self._get_status_int() // 100 == 2:
start_response(save_response_status, self._response_headers,
self._response_exc_info)
@ -190,21 +191,6 @@ class _StaticWebContext(WSGIContext):
save_response_exc_info)
return response
def _get_escalated_env(self, env):
"""
Returns a new fresh WSGI environment with escalated privileges to do
backend checks, listings, etc. that the remote user wouldn't be able to
accomplish directly.
"""
new_env = {'REQUEST_METHOD': 'GET',
'HTTP_USER_AGENT': '%s StaticWeb' % env.get('HTTP_USER_AGENT')}
for name in ('eventlet.posthooks', 'swift.trans_id', 'REMOTE_USER',
'SCRIPT_NAME', 'SERVER_NAME', 'SERVER_PORT',
'SERVER_PROTOCOL', 'swift.cache'):
if name in env:
new_env[name] = env[name]
return new_env
def _get_container_info(self, env):
"""
Retrieves x-container-meta-web-index, x-container-meta-web-error,
@ -224,11 +210,9 @@ class _StaticWebContext(WSGIContext):
(self._index, self._error, self._listings,
self._listings_css) = cached_data
return
tmp_env = self._get_escalated_env(env)
tmp_env['REQUEST_METHOD'] = 'HEAD'
req = Request.blank('/%s/%s/%s' % (self.version, self.account,
self.container), environ=tmp_env)
resp = req.get_response(self.app)
resp = make_pre_authed_request(env, 'HEAD',
'/%s/%s/%s' % (self.version, self.account, self.container),
agent=self.agent).get_response(self.app)
if resp.status_int // 100 == 2:
self._index = \
resp.headers.get('x-container-meta-web-index', '').strip()
@ -256,10 +240,9 @@ class _StaticWebContext(WSGIContext):
if self._listings.lower() not in TRUE_VALUES:
resp = HTTPNotFound()(env, self._start_response)
return self._error_response(resp, env, start_response)
tmp_env = self._get_escalated_env(env)
tmp_env['REQUEST_METHOD'] = 'GET'
tmp_env['PATH_INFO'] = \
'/%s/%s/%s' % (self.version, self.account, self.container)
tmp_env = make_pre_authed_env(env, 'GET',
'/%s/%s/%s' % (self.version, self.account, self.container),
self.agent)
tmp_env['QUERY_STRING'] = 'delimiter=/&format=json'
if prefix:
tmp_env['QUERY_STRING'] += '&prefix=%s' % quote(prefix)
@ -431,10 +414,10 @@ class _StaticWebContext(WSGIContext):
return resp
if status_int == 404:
if env['PATH_INFO'][-1] != '/':
tmp_env = self._get_escalated_env(env)
tmp_env['REQUEST_METHOD'] = 'GET'
tmp_env['PATH_INFO'] = '/%s/%s/%s' % (self.version,
self.account, self.container)
tmp_env = make_pre_authed_env(env, 'GET',
'/%s/%s/%s' % (self.version, self.account,
self.container),
self.agent)
tmp_env['QUERY_STRING'] = 'limit=1&format=json&delimiter' \
'=/&limit=1&prefix=%s' % quote(self.obj + '/')
resp = self._app_call(tmp_env)

26
swift/common/middleware/tempurl.py
View File

@ -87,6 +87,7 @@ from urllib import quote, unquote
from urlparse import parse_qs
from swift.common.utils import get_logger
from swift.common.wsgi import make_pre_authed_env
#: Default headers to remove from incoming requests. Simply a whitespace
@ -211,6 +212,8 @@ class TempURL(object):
#: Lowercase, like `x-matches-remove-prefix-but-okay-*`.
self.outgoing_allow_headers_startswith = \
[h[:-1] for h in headers if h[-1] == '*']
#: HTTP user agent to use for subrequests.
self.agent = '%(orig)s TempURL'
def __call__(self, env, start_response):
"""
@ -321,19 +324,10 @@ class TempURL(object):
if memcache:
key = memcache.get('temp-url-key/%s' % account)
if not key:
newenv = {'REQUEST_METHOD': 'HEAD', 'SCRIPT_NAME': '',
'PATH_INFO': '/v1/' + account, 'CONTENT_LENGTH': '0',
'SERVER_PROTOCOL': 'HTTP/1.0',
'HTTP_USER_AGENT': 'TempURL', 'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http', 'wsgi.input': StringIO('')}
for name in ('SERVER_NAME', 'SERVER_PORT', 'wsgi.errors',
'wsgi.multithread', 'wsgi.multiprocess',
'wsgi.run_once', 'swift.cache', 'swift.trans_id'):
if name in env:
newenv[name] = env[name]
newenv['swift.authorize'] = lambda req: None
newenv['swift.authorize_override'] = True
newenv['REMOTE_USER'] = '.wsgi.tempurl'
newenv = make_pre_authed_env(env, 'HEAD', '/v1/' + account,
self.agent)
newenv['CONTENT_LENGTH'] = '0'
newenv['wsgi.input'] = StringIO('')
key = [None]
def _start_response(status, response_headers, exc_info=None):
@ -341,7 +335,11 @@ class TempURL(object):
if h.lower() == 'x-account-meta-temp-url-key':
key[0] = v
self.app(newenv, _start_response)
i = iter(self.app(newenv, _start_response))
try:
i.next()
except StopIteration:
pass
key = key[0]
if key and memcache:
memcache.set('temp-url-key/%s' % account, key, timeout=60)

83
swift/common/wsgi.py
View File

@ -224,12 +224,12 @@ class WSGIContext(object):
Ensures start_response has been called before returning.
"""
resp = iter(self.app(env, self._start_response))
first_chunk = []
try:
first_chunk.append(resp.next())
first_chunk = resp.next()
except StopIteration:
pass
return chain(first_chunk, resp)
return iter([])
else: # We got a first_chunk
return chain([first_chunk], resp)
def _get_status_int(self):
"""
@ -246,27 +246,29 @@ class WSGIContext(object):
return None
def make_pre_authed_request(env, method, path, body=None, headers=None,
agent='Swift'):
def make_pre_authed_request(env, method=None, path=None, body=None,
headers=None, agent='Swift'):
"""
Makes a new webob.Request based on the current env but with the
parameters specified. Note that this request will be preauthorized.
:param env: Current WSGI environment dictionary
:param method: HTTP method of new request
:param path: HTTP path of new request
:param body: HTTP body of new request; None by default
:param headers: Extra HTTP headers of new request; None by default
:returns: webob.Request object
(Stolen from Swauth: https://github.com/gholt/swauth)
:param env: The WSGI environment to base the new request on.
:param method: HTTP method of new request; default is from
the original env.
:param path: HTTP path of new request; default is from the
original env.
:param body: HTTP body of new request; empty by default.
:param headers: Extra HTTP headers of new request; None by
default.
:param agent: The HTTP user agent to use; default 'Swift'. You
can put %(orig)s in the agent to have it replaced
with the original env's HTTP_USER_AGENT, such as
'%(orig)s StaticWeb'. You also set agent to None to
use the original env's HTTP_USER_AGENT or '' to
have no HTTP_USER_AGENT.
:returns: Fresh webob.Request object.
"""
newenv = {'REQUEST_METHOD': method, 'HTTP_USER_AGENT': agent}
for name in ('swift.cache', 'swift.trans_id'):
if name in env:
newenv[name] = env[name]
newenv['swift.authorize'] = lambda req: None
newenv = make_pre_authed_env(env, method, path, agent)
if not headers:
headers = {}
if body:
@ -274,3 +276,44 @@ def make_pre_authed_request(env, method, path, body=None, headers=None,
headers=headers)
else:
return Request.blank(path, environ=newenv, headers=headers)
def make_pre_authed_env(env, method=None, path=None, agent='Swift'):
"""
Returns a new fresh WSGI environment with escalated privileges to
do backend checks, listings, etc. that the remote user wouldn't
be able to accomplish directly.
:param env: The WSGI environment to base the new environment on.
:param method: The new REQUEST_METHOD or None to use the
original.
:param path: The new PATH_INFO or None to use the original.
:param agent: The HTTP user agent to use; default 'Swift'. You
can put %(orig)s in the agent to have it replaced
with the original env's HTTP_USER_AGENT, such as
'%(orig)s StaticWeb'. You also set agent to None to
use the original env's HTTP_USER_AGENT or '' to
have no HTTP_USER_AGENT.
:returns: Fresh WSGI environment.
"""
newenv = {}
for name in ('eventlet.posthooks', 'HTTP_USER_AGENT',
'PATH_INFO', 'REMOTE_USER', 'REQUEST_METHOD',
'SCRIPT_NAME', 'SERVER_NAME', 'SERVER_PORT',
'SERVER_PROTOCOL', 'swift.cache', 'swift.source',
'swift.trans_id'):
if name in env:
newenv[name] = env[name]
if method:
newenv['REQUEST_METHOD'] = method
if path:
newenv['PATH_INFO'] = path
if agent:
newenv['HTTP_USER_AGENT'] = (
agent % {'orig': env.get('HTTP_USER_AGENT', '')}).strip()
elif agent == '' and 'HTTP_USER_AGENT' in newenv:
del newenv['HTTP_USER_AGENT']
newenv['swift.authorize'] = lambda req: None
newenv['swift.authorize_override'] = True
newenv['REMOTE_USER'] = '.wsgi.pre_authed'
return newenv

2
test/unit/common/middleware/test_formpost.py
View File

@ -74,7 +74,7 @@ class FakeApp(object):
env['wsgi.input'] = StringIO(body)
self.requests.append(Request.blank('', environ=env))
if env.get('swift.authorize_override') and \
env.get('REMOTE_USER') != '.wsgi.formpost':
env.get('REMOTE_USER') != '.wsgi.pre_authed':
raise Exception('Invalid REMOTE_USER %r with '
'swift.authorize_override' % (env.get('REMOTE_USER'),))
if 'swift.authorize' in env: