From eac721b003ae16931f0deb1f1457d5cbca263d74 Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 18 Feb 2011 23:22:15 -0800 Subject: [PATCH 01/27] Working staticweb filter --- setup.py | 1 + swift/common/middleware/staticweb.py | 236 +++++++++++++++++++++++++++ swift/proxy/server.py | 2 +- 3 files changed, 238 insertions(+), 1 deletion(-) create mode 100644 swift/common/middleware/staticweb.py diff --git a/setup.py b/setup.py index c80d62ddc8..3cbbb324e5 100644 --- a/setup.py +++ b/setup.py @@ -120,6 +120,7 @@ setup( 'catch_errors=swift.common.middleware.catch_errors:filter_factory', 'domain_remap=swift.common.middleware.domain_remap:filter_factory', 'swift3=swift.common.middleware.swift3:filter_factory', + 'staticweb=swift.common.middleware.staticweb:filter_factory', ], }, ) diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py new file mode 100644 index 0000000000..1484da39af --- /dev/null +++ b/swift/common/middleware/staticweb.py @@ -0,0 +1,236 @@ +# Copyright (c) 2010-2011 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + import simplejson as json +except ImportError: + import json + +import cgi +import urllib + +from webob import Response +from webob.exc import HTTPMovedPermanently, HTTPNotFound, HTTPUnauthorized + +from swift.common.utils import split_path + + +# To use: +# Put the staticweb filter just after the auth filter. +# Make the container publicly readable: +# st post -r '.r:*' container +# You should be able to get objects now, but not do listings. +# Set an index file: +# st post -m 'index:index.html' container +# You should be able to hit path's that have an index.html without needing to +# type the index.html part, but still not do listings. +# Turn on listings: +# st post -m 'index:allow_listings,index.html' container +# Set an error file: +# st post -m 'error:error.html' container +# Now 404's should load 404error.html + +# TODO: Tests +# TODO: Docs +# TODO: Accept header negotiation: make static web work with authed as well +# TODO: get_container_info can be memcached + +class StaticWeb(object): + + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def start_response(self, status, headers, exc_info=None): + self.response_status = status + self.response_headers = headers + self.response_exc_info = exc_info + + def error_response(self, response, env, start_response): + if not self.error: + start_response(self.response_status, self.response_headers, + self.response_exc_info) + return response + tmp_env = dict(env) + self.strip_ifs(tmp_env) + tmp_env['PATH_INFO'] = '/%s/%s/%s/%s%s' % (self.version, self.account, + self.container, self.get_status_int(), self.error) + tmp_env['REQUEST_METHOD'] = 'GET' + return self.app(tmp_env, start_response) + + def get_status_int(self): + return int(self.response_status.split(' ', 1)[0]) + + def get_header(self, headers, name, default_value=None): + for header, value in headers: + if header.lower() == name: + return value + return default_value + + def strip_ifs(self, env): + for key in [k for k in env.keys() if k.startswith('HTTP_IF_')]: + del env[key] + + def get_container_info(self, env, start_response): + self.index = self.error = None + self.index_allow_listings = False + tmp_env = dict(env) + self.strip_ifs(tmp_env) + tmp_env['REQUEST_METHOD'] = 'HEAD' + tmp_env['PATH_INFO'] = \ + '/%s/%s/%s' % (self.version, self.account, self.container) + resp = self.app(tmp_env, self.start_response) + if self.get_status_int() // 100 != 2: + return + self.index = self.get_header(self.response_headers, + 'x-container-meta-index', '').strip() + self.error = self.get_header(self.response_headers, + 'x-container-meta-error', '').strip() + if not self.index: + return + if self.index.lower() == 'allow_listings': + self.index_allow_listings = True + elif self.index.lower().startswith('allow_listings,'): + self.index = self.index[len('allow_listings,'):] + self.index_allow_listings = True + + def listing(self, env, start_response, prefix=None): + tmp_env = dict(env) + self.strip_ifs(tmp_env) + tmp_env['REQUEST_METHOD'] = 'GET' + tmp_env['PATH_INFO'] = \ + '/%s/%s/%s' % (self.version, self.account, self.container) + tmp_env['QUERY_STRING'] = 'delimiter=/&format=json' + if prefix: + tmp_env['QUERY_STRING'] += '&prefix=%s' % urllib.quote(prefix) + resp = self.app(tmp_env, self.start_response) + if self.get_status_int() // 100 != 2: + return self.error_response(resp, env, start_response) + listing = json.loads(''.join(resp)) + if not listing: + resp = HTTPNotFound()(env, self.start_response) + return self.error_response(resp, env, start_response) + headers = {'Content-Type': 'text/html'} + body = 'Listing of%s' \ + '

Listing of %s

\n' % \ + (cgi.escape(env['PATH_INFO']), cgi.escape(env['PATH_INFO'])) + if prefix: + body += '../
' + for item in listing: + if 'subdir' in item: + subdir = item['subdir'] + if prefix: + subdir = subdir[len(prefix):] + body += '%s
' % \ + (urllib.quote(subdir), cgi.escape(subdir)) + for item in listing: + if 'name' in item: + name = item['name'] + if prefix: + name = name[len(prefix):] + body += '%s
' % \ + (urllib.quote(name), cgi.escape(name)) + body += '

\n' + return Response(headers=headers, body=body)(env, start_response) + + def handle_container(self, env, start_response): + self.get_container_info(env, start_response) + if not self.index and not self.index_allow_listings: + resp = HTTPNotFound()(env, self.start_response) + return self.error_response(resp, env, start_response) + if env['PATH_INFO'][-1] != '/': + return HTTPMovedPermanently( + location=env['PATH_INFO'] + '/')(env, start_response) + if not self.index and self.index_allow_listings: + return self.listing(env, start_response) + tmp_env = dict(env) + tmp_env['PATH_INFO'] += self.index + resp = self.app(tmp_env, self.start_response) + status_int = self.get_status_int() + if status_int == 404 and self.index_allow_listings: + return self.listing(env, start_response) + elif self.get_status_int() // 100 not in (2, 3): + return self.error_response(resp, env, start_response) + start_response(self.response_status, self.response_headers, + self.response_exc_info) + return resp + + def handle_object(self, env, start_response): + tmp_env = dict(env) + resp = self.app(tmp_env, self.start_response) + status_int = self.get_status_int() + if status_int // 100 in (2, 3): + start_response(self.response_status, self.response_headers, + self.response_exc_info) + return resp + if status_int != 404: + return self.error_response(resp, env, start_response) + self.get_container_info(env, start_response) + if not self.index and not self.index_allow_listings: + return self.app(env, start_response) + if self.index: + tmp_env = dict(env) + if tmp_env['PATH_INFO'][-1] != '/': + tmp_env['PATH_INFO'] += '/' + tmp_env['PATH_INFO'] += self.index + resp = self.app(tmp_env, self.start_response) + status_int = self.get_status_int() + if status_int // 100 in (2, 3): + if env['PATH_INFO'][-1] != '/': + return HTTPMovedPermanently( + location=env['PATH_INFO'] + '/')(env, start_response) + start_response(self.response_status, self.response_headers, + self.response_exc_info) + return resp + elif status_int == 404 and self.index_allow_listings: + if env['PATH_INFO'][-1] != '/': + tmp_env = dict(env) + self.strip_ifs(tmp_env) + tmp_env['REQUEST_METHOD'] = 'GET' + tmp_env['PATH_INFO'] = '/%s/%s/%s' % (self.version, + self.account, self.container) + tmp_env['QUERY_STRING'] = 'limit=1&format=json&delimiter' \ + '=/&limit=1&prefix=%s' % urllib.quote(self.obj + '/') + resp = self.app(tmp_env, self.start_response) + if self.get_status_int() // 100 != 2 or \ + not json.loads(''.join(resp)): + resp = HTTPNotFound()(env, self.start_response) + return self.error_response(resp, env, start_response) + return HTTPMovedPermanently(location=env['PATH_INFO'] + + '/')(env, start_response) + return self.listing(env, start_response, self.obj) + return self.app(env, start_response) + + def __call__(self, env, start_response): + if env.get('REMOTE_USER') or \ + env['REQUEST_METHOD'] not in ('HEAD', 'GET'): + return self.app(env, start_response) + (self.version, self.account, self.container, self.obj) = \ + split_path(env['PATH_INFO'], 2, 4, True) + if self.obj: + return self.handle_object(env, start_response) + elif self.container: + return self.handle_container(env, start_response) + return self.app(env, start_response) + + +def filter_factory(global_conf, **local_conf): + """Returns a WSGI filter app for use with paste.deploy.""" + conf = global_conf.copy() + conf.update(local_conf) + + def staticweb_filter(app): + return StaticWeb(app, conf) + return staticweb_filter diff --git a/swift/proxy/server.py b/swift/proxy/server.py index cc09ac1459..ad1fd75557 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -545,7 +545,7 @@ class Controller(object): status_index = statuses.index(status) resp.status = '%s %s' % (status, reasons[status_index]) resp.body = bodies[status_index] - resp.content_type = 'text/plain' + resp.content_type = 'text/html' if etag: resp.headers['etag'] = etag.strip('"') return resp From 6639b0ec0c4b65270a08af6e5cb40c134219eac9 Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 18 Feb 2011 23:37:51 -0800 Subject: [PATCH 02/27] staticweb: Reallow direct container listings --- swift/common/middleware/staticweb.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index 1484da39af..8936728b04 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -44,8 +44,13 @@ from swift.common.utils import split_path # TODO: Tests # TODO: Docs +# TODO: Make a disallow_listings to restrict direct container listings. These +# have to stay on by default because of swauth and any other similar +# middleware. The lower level indirect listings can stay disabled by +# default. # TODO: Accept header negotiation: make static web work with authed as well # TODO: get_container_info can be memcached +# TODO: Blueprint class StaticWeb(object): @@ -148,8 +153,7 @@ class StaticWeb(object): def handle_container(self, env, start_response): self.get_container_info(env, start_response) if not self.index and not self.index_allow_listings: - resp = HTTPNotFound()(env, self.start_response) - return self.error_response(resp, env, start_response) + return self.app(env, start_response) if env['PATH_INFO'][-1] != '/': return HTTPMovedPermanently( location=env['PATH_INFO'] + '/')(env, start_response) From 469c287b5ba201a567443bb562256dccbae9abd8 Mon Sep 17 00:00:00 2001 From: gholt Date: Sat, 19 Feb 2011 12:35:58 -0800 Subject: [PATCH 03/27] staticweb: work in progress --- swift/common/middleware/acl.py | 5 + swift/common/middleware/staticweb.py | 156 +++++++++++++-------------- swift/common/middleware/swauth.py | 4 +- 3 files changed, 85 insertions(+), 80 deletions(-) diff --git a/swift/common/middleware/acl.py b/swift/common/middleware/acl.py index f08780eedb..e31ebda743 100644 --- a/swift/common/middleware/acl.py +++ b/swift/common/middleware/acl.py @@ -58,6 +58,10 @@ def clean_acl(name, value): .r: .r:- + By default, allowing read access via .r will allow listing objects in the + container as well as retrieving objects from the container. To turn off + listings, use the .rnolisting directive. + Also, .r designations aren't allowed in headers whose names include the word 'write'. @@ -71,6 +75,7 @@ def clean_acl(name, value): ``bob,,,sue`` ``bob,sue`` ``.referrer : *`` ``.r:*`` ``.ref:*.example.com`` ``.r:.example.com`` + ``.r:*, .rnolisting`` ``.r:*,.rnolisting`` ====================== ====================== :param name: The name of the header being cleaned, such as X-Container-Read diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index 8936728b04..d9fdc196af 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -21,36 +21,37 @@ except ImportError: import cgi import urllib -from webob import Response +from webob import Response, Request from webob.exc import HTTPMovedPermanently, HTTPNotFound, HTTPUnauthorized -from swift.common.utils import split_path +from swift.common.utils import split_path, TRUE_VALUES # To use: # Put the staticweb filter just after the auth filter. # Make the container publicly readable: # st post -r '.r:*' container -# You should be able to get objects now, but not do listings. -# Set an index file: +# You should be able to get objects and do direct container listings +# now, though they'll be in the REST API format. +# Set an index file directive: # st post -m 'index:index.html' container -# You should be able to hit path's that have an index.html without needing to -# type the index.html part, but still not do listings. -# Turn on listings: -# st post -m 'index:allow_listings,index.html' container +# You should be able to hit paths that have an index.html without +# needing to type the index.html part and listings will now be HTML. +# Turn off listings: +# st post -r '.r:*,.rnolisting' container # Set an error file: # st post -m 'error:error.html' container -# Now 404's should load 404error.html - -# TODO: Tests -# TODO: Docs -# TODO: Make a disallow_listings to restrict direct container listings. These -# have to stay on by default because of swauth and any other similar -# middleware. The lower level indirect listings can stay disabled by -# default. -# TODO: Accept header negotiation: make static web work with authed as well -# TODO: get_container_info can be memcached -# TODO: Blueprint +# Now 401's should load s 401error.html, 404's should load +# 404error.html, etc. +# +# This mode is normally only active for anonymous requests. If you +# want to use it with authenticated requests, set the X-Web-Mode: +# true header. +# +# TODO: Tests. +# TODO: Docs. +# TODO: get_container_info can be memcached. +# TODO: Blueprint. class StaticWeb(object): @@ -68,12 +69,22 @@ class StaticWeb(object): start_response(self.response_status, self.response_headers, self.response_exc_info) return response + save_response_status = self.response_status + save_response_headers = self.response_headers + save_response_exc_info = self.response_exc_info tmp_env = dict(env) self.strip_ifs(tmp_env) tmp_env['PATH_INFO'] = '/%s/%s/%s/%s%s' % (self.version, self.account, self.container, self.get_status_int(), self.error) tmp_env['REQUEST_METHOD'] = 'GET' - return self.app(tmp_env, start_response) + resp = self.app(tmp_env, self.start_response) + if self.get_status_int() // 100 == 2: + start_response(self.response_status, self.response_headers, + self.response_exc_info) + return resp + start_response(save_response_status, save_response_headers, + save_response_exc_info) + return response def get_status_int(self): return int(self.response_status.split(' ', 1)[0]) @@ -90,26 +101,16 @@ class StaticWeb(object): def get_container_info(self, env, start_response): self.index = self.error = None - self.index_allow_listings = False - tmp_env = dict(env) - self.strip_ifs(tmp_env) - tmp_env['REQUEST_METHOD'] = 'HEAD' - tmp_env['PATH_INFO'] = \ - '/%s/%s/%s' % (self.version, self.account, self.container) - resp = self.app(tmp_env, self.start_response) - if self.get_status_int() // 100 != 2: - return - self.index = self.get_header(self.response_headers, - 'x-container-meta-index', '').strip() - self.error = self.get_header(self.response_headers, - 'x-container-meta-error', '').strip() - if not self.index: - return - if self.index.lower() == 'allow_listings': - self.index_allow_listings = True - elif self.index.lower().startswith('allow_listings,'): - self.index = self.index[len('allow_listings,'):] - self.index_allow_listings = True + tmp_env = {'REQUEST_METHOD': 'HEAD', 'HTTP_USER_AGENT': 'StaticWeb'} + for name in ('swift.cache', 'HTTP_X_CF_TRANS_ID'): + if name in env: + tmp_env[name] = env[name] + req = Request.blank('/%s/%s/%s' % (self.version, self.account, + self.container), environ=tmp_env) + resp = req.get_response(self.app) + if resp.status_int // 100 == 2: + self.index = resp.headers.get('x-container-meta-index', '').strip() + self.error = resp.headers.get('x-container-meta-error', '').strip() def listing(self, env, start_response, prefix=None): tmp_env = dict(env) @@ -152,18 +153,16 @@ class StaticWeb(object): def handle_container(self, env, start_response): self.get_container_info(env, start_response) - if not self.index and not self.index_allow_listings: + if not self.index: return self.app(env, start_response) if env['PATH_INFO'][-1] != '/': return HTTPMovedPermanently( location=env['PATH_INFO'] + '/')(env, start_response) - if not self.index and self.index_allow_listings: - return self.listing(env, start_response) tmp_env = dict(env) tmp_env['PATH_INFO'] += self.index resp = self.app(tmp_env, self.start_response) status_int = self.get_status_int() - if status_int == 404 and self.index_allow_listings: + if status_int == 404: return self.listing(env, start_response) elif self.get_status_int() // 100 not in (2, 3): return self.error_response(resp, env, start_response) @@ -182,44 +181,43 @@ class StaticWeb(object): if status_int != 404: return self.error_response(resp, env, start_response) self.get_container_info(env, start_response) - if not self.index and not self.index_allow_listings: + if not self.index: return self.app(env, start_response) - if self.index: - tmp_env = dict(env) - if tmp_env['PATH_INFO'][-1] != '/': - tmp_env['PATH_INFO'] += '/' - tmp_env['PATH_INFO'] += self.index - resp = self.app(tmp_env, self.start_response) - status_int = self.get_status_int() - if status_int // 100 in (2, 3): - if env['PATH_INFO'][-1] != '/': - return HTTPMovedPermanently( - location=env['PATH_INFO'] + '/')(env, start_response) - start_response(self.response_status, self.response_headers, - self.response_exc_info) - return resp - elif status_int == 404 and self.index_allow_listings: - if env['PATH_INFO'][-1] != '/': - tmp_env = dict(env) - self.strip_ifs(tmp_env) - tmp_env['REQUEST_METHOD'] = 'GET' - tmp_env['PATH_INFO'] = '/%s/%s/%s' % (self.version, - self.account, self.container) - tmp_env['QUERY_STRING'] = 'limit=1&format=json&delimiter' \ - '=/&limit=1&prefix=%s' % urllib.quote(self.obj + '/') - resp = self.app(tmp_env, self.start_response) - if self.get_status_int() // 100 != 2 or \ - not json.loads(''.join(resp)): - resp = HTTPNotFound()(env, self.start_response) - return self.error_response(resp, env, start_response) - return HTTPMovedPermanently(location=env['PATH_INFO'] + - '/')(env, start_response) - return self.listing(env, start_response, self.obj) - return self.app(env, start_response) + tmp_env = dict(env) + if tmp_env['PATH_INFO'][-1] != '/': + tmp_env['PATH_INFO'] += '/' + tmp_env['PATH_INFO'] += self.index + resp = self.app(tmp_env, self.start_response) + status_int = self.get_status_int() + if status_int // 100 in (2, 3): + if env['PATH_INFO'][-1] != '/': + return HTTPMovedPermanently( + location=env['PATH_INFO'] + '/')(env, start_response) + start_response(self.response_status, self.response_headers, + self.response_exc_info) + return resp + elif status_int == 404: + if env['PATH_INFO'][-1] != '/': + tmp_env = dict(env) + self.strip_ifs(tmp_env) + tmp_env['REQUEST_METHOD'] = 'GET' + tmp_env['PATH_INFO'] = '/%s/%s/%s' % (self.version, + self.account, self.container) + tmp_env['QUERY_STRING'] = 'limit=1&format=json&delimiter' \ + '=/&limit=1&prefix=%s' % urllib.quote(self.obj + '/') + resp = self.app(tmp_env, self.start_response) + if self.get_status_int() // 100 != 2 or \ + not json.loads(''.join(resp)): + resp = HTTPNotFound()(env, self.start_response) + return self.error_response(resp, env, start_response) + return HTTPMovedPermanently(location=env['PATH_INFO'] + + '/')(env, start_response) + return self.listing(env, start_response, self.obj) def __call__(self, env, start_response): - if env.get('REMOTE_USER') or \ - env['REQUEST_METHOD'] not in ('HEAD', 'GET'): + if env['REQUEST_METHOD'] not in ('HEAD', 'GET') or \ + (env.get('REMOTE_USER') and + not env.get('HTTP_X_WEB_MODE', '') in TRUE_VALUES): return self.app(env, start_response) (self.version, self.account, self.container, self.obj) = \ split_path(env['PATH_INFO'], 2, 4, True) diff --git a/swift/common/middleware/swauth.py b/swift/common/middleware/swauth.py index 68b0d7afaf..86cdcac07c 100644 --- a/swift/common/middleware/swauth.py +++ b/swift/common/middleware/swauth.py @@ -277,6 +277,8 @@ class Swauth(object): return None referrers, groups = parse_acl(getattr(req, 'acl', None)) if referrer_allowed(req.referer, referrers): + if not obj and '.rnolisting' in groups: + return HTTPUnauthorized(request=req) return None if not req.remote_user: return self.denied_response(req) @@ -1179,7 +1181,7 @@ class Swauth(object): :returns: webob.Request object """ - newenv = {'REQUEST_METHOD': method} + newenv = {'REQUEST_METHOD': method, 'HTTP_USER_AGENT': 'Swauth'} for name in ('swift.cache', 'HTTP_X_CF_TRANS_ID'): if name in env: newenv[name] = env[name] From 2491c6470303c608756bf124a2f3e099ae1dd3f2 Mon Sep 17 00:00:00 2001 From: gholt Date: Sat, 19 Feb 2011 13:09:20 -0800 Subject: [PATCH 04/27] staticweb: added a todo --- swift/common/middleware/staticweb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index d9fdc196af..319d6a28f4 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -48,6 +48,7 @@ from swift.common.utils import split_path, TRUE_VALUES # want to use it with authenticated requests, set the X-Web-Mode: # true header. # +# TODO: Make new headers instead of using user metadata. # TODO: Tests. # TODO: Docs. # TODO: get_container_info can be memcached. From 765f5b5c3f058828eb54fffecac6f162541de97f Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Sun, 20 Mar 2011 22:14:03 +0000 Subject: [PATCH 05/27] refactor obj-rep a bit and move hash recalculate to before rsync step --- swift/obj/replicator.py | 63 +++++++++----------------------- swift/obj/server.py | 9 ++--- test/unit/obj/test_replicator.py | 17 +++++++++ 3 files changed, 37 insertions(+), 52 deletions(-) diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index 4339493a99..34fde2dc42 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -85,37 +85,6 @@ def hash_suffix(path, reclaim_age): return md5.hexdigest() -def recalculate_hashes(partition_dir, suffixes, reclaim_age=ONE_WEEK): - """ - Recalculates hashes for the given suffixes in the partition and updates - them in the partition's hashes file. - - :param partition_dir: directory of the partition in which to recalculate - :param suffixes: list of suffixes to recalculate - :param reclaim_age: age in seconds at which tombstones should be removed - """ - - def tpool_listdir(partition_dir): - return dict(((suff, None) for suff in os.listdir(partition_dir) - if len(suff) == 3 and isdir(join(partition_dir, suff)))) - hashes_file = join(partition_dir, HASH_FILE) - with lock_path(partition_dir): - try: - with open(hashes_file, 'rb') as fp: - hashes = pickle.load(fp) - except Exception: - hashes = tpool.execute(tpool_listdir, partition_dir) - for suffix in suffixes: - suffix_dir = join(partition_dir, suffix) - if os.path.exists(suffix_dir): - hashes[suffix] = hash_suffix(suffix_dir, reclaim_age) - elif suffix in hashes: - del hashes[suffix] - with open(hashes_file + '.tmp', 'wb') as fp: - pickle.dump(hashes, fp, PICKLE_PROTOCOL) - renamer(hashes_file + '.tmp', hashes_file) - - def invalidate_hash(suffix_dir): """ Invalidates the hash for a suffix_dir in the partition's hashes file. @@ -141,23 +110,21 @@ def invalidate_hash(suffix_dir): renamer(hashes_file + '.tmp', hashes_file) -def get_hashes(partition_dir, do_listdir=True, reclaim_age=ONE_WEEK): +def get_hashes(partition_dir, recalculate=[], do_listdir=False, + reclaim_age=ONE_WEEK): """ Get a list of hashes for the suffix dir. do_listdir causes it to mistrust the hash cache for suffix existence at the (unexpectedly high) cost of a listdir. reclaim_age is just passed on to hash_suffix. :param partition_dir: absolute path of partition to get hashes for + :param recalculate: list of suffixes which should be recalculated when got :param do_listdir: force existence check for all hashes in the partition :param reclaim_age: age at which to remove tombstones :returns: tuple of (number of suffix dirs hashed, dictionary of hashes) """ - def tpool_listdir(hashes, partition_dir): - return dict(((suff, hashes.get(suff, None)) - for suff in os.listdir(partition_dir) - if len(suff) == 3 and isdir(join(partition_dir, suff)))) hashed = 0 hashes_file = join(partition_dir, HASH_FILE) with lock_path(partition_dir): @@ -169,8 +136,12 @@ def get_hashes(partition_dir, do_listdir=True, reclaim_age=ONE_WEEK): except Exception: do_listdir = True if do_listdir: - hashes = tpool.execute(tpool_listdir, hashes, partition_dir) + hashes = dict(((suff, hashes.get(suff, None)) + for suff in os.listdir(partition_dir) + if len(suff) == 3 and isdir(join(partition_dir, suff)))) modified = True + for hash_ in recalculate: + hashes[hash_] = None for suffix, hash_ in hashes.items(): if not hash_: suffix_dir = join(partition_dir, suffix) @@ -342,8 +313,7 @@ class ObjectReplicator(Daemon): success = self.rsync(node, job, suffixes) if success: with Timeout(self.http_timeout): - http_connect(node['ip'], - node['port'], + http_connect(node['ip'], node['port'], node['device'], job['partition'], 'REPLICATE', '/' + '-'.join(suffixes), headers={'Content-Length': '0'}).getresponse().read() @@ -366,7 +336,7 @@ class ObjectReplicator(Daemon): self.replication_count += 1 begin = time.time() try: - hashed, local_hash = get_hashes(job['path'], + hashed, local_hash = tpool.execute(get_hashes, job['path'], do_listdir=(self.replication_count % 10) == 0, reclaim_age=self.reclaim_age) self.suffix_hash += hashed @@ -394,14 +364,15 @@ class ObjectReplicator(Daemon): continue remote_hash = pickle.loads(resp.read()) del resp - suffixes = [suffix for suffix in local_hash - if local_hash[suffix] != - remote_hash.get(suffix, -1)] + suffixes = [suffix for suffix in local_hash if + local_hash[suffix] != remote_hash.get(suffix, -1)] if not suffixes: continue + hashed, local_hash = tpool.execute(get_hashes, job['path'], + recalculate=suffixes, reclaim_age=self.reclaim_age) + suffixes = [suffix for suffix in local_hash if + local_hash[suffix] != remote_hash.get(suffix, -1)] self.rsync(node, job, suffixes) - recalculate_hashes(job['path'], suffixes, - reclaim_age=self.reclaim_age) with Timeout(self.http_timeout): conn = http_connect(node['ip'], node['port'], node['device'], job['partition'], 'REPLICATE', @@ -556,7 +527,7 @@ class ObjectReplicator(Daemon): _("Object replication complete. (%.02f minutes)"), total) def run_forever(self, *args, **kwargs): - self.logger.info("Starting object replicator in daemon mode.") + self.logger.info(_("Starting object replicator in daemon mode.")) # Run the replicator continually while True: start = time.time() diff --git a/swift/obj/server.py b/swift/obj/server.py index 7d4b433a01..5e8f6e2cd1 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -42,8 +42,7 @@ from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, check_mount, \ check_float, check_utf8 from swift.common.exceptions import ConnectionTimeout -from swift.obj.replicator import get_hashes, invalidate_hash, \ - recalculate_hashes +from swift.obj.replicator import get_hashes, invalidate_hash DATADIR = 'objects' @@ -574,10 +573,8 @@ class ObjectController(object): path = os.path.join(self.devices, device, DATADIR, partition) if not os.path.exists(path): mkdirs(path) - if suffix: - recalculate_hashes(path, suffix.split('-')) - return Response() - _junk, hashes = get_hashes(path, do_listdir=False) + suffixes = suffix.split('-') if suffix else [] + _junk, hashes = tpool.execute(get_hashes, path, recalculate=suffixes) return Response(body=pickle.dumps(hashes)) def __call__(self, env, start_response): diff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py index 9558a2b0d8..290077b32c 100644 --- a/test/unit/obj/test_replicator.py +++ b/test/unit/obj/test_replicator.py @@ -188,6 +188,23 @@ class TestObjectReplicator(unittest.TestCase): object_replicator.http_connect = was_connector + def test_get_hashes(self): + df = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o') + mkdirs(df.datadir) + with open(os.path.join(df.datadir, normalize_timestamp( + time.time()) + '.ts'), 'wb') as f: + f.write('1234567890') + part = os.path.join(self.objects, '0') + hashed, hashes = object_replicator.get_hashes(part) + self.assertEquals(hashed, 1) + self.assert_('a83' in hashes) + hashed, hashes = object_replicator.get_hashes(part, do_listdir=True) + self.assertEquals(hashed, 0) + self.assert_('a83' in hashes) + hashed, hashes = object_replicator.get_hashes(part, recalculate=['a83']) + self.assertEquals(hashed, 1) + self.assert_('a83' in hashes) + def test_hash_suffix_one_file(self): df = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o') mkdirs(df.datadir) From 06094af35942be08807a60d732fbe00ba4b30f3d Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Tue, 22 Mar 2011 18:17:47 -0500 Subject: [PATCH 06/27] fixed object POST so content encoding can be set and deleted --- swift/obj/server.py | 3 +++ test/unit/obj/test_server.py | 22 ++++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/swift/obj/server.py b/swift/obj/server.py index 7d4b433a01..0914371a33 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -353,6 +353,9 @@ class ObjectController(object): metadata = {'X-Timestamp': request.headers['x-timestamp']} metadata.update(val for val in request.headers.iteritems() if val[0].lower().startswith('x-object-meta-')) + if 'content-encoding' in request.headers: + metadata['Content-Encoding'] = \ + request.headers['Content-Encoding'] with file.mkstemp() as (fd, tmppath): file.put(fd, tmppath, metadata, extension='.meta') return response_class(request=request) diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 22d2fe20a4..d91ae85cf2 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -73,6 +73,7 @@ class TestObjectController(unittest.TestCase): headers={'X-Timestamp': timestamp, 'X-Object-Meta-3': 'Three', 'X-Object-Meta-4': 'Four', + 'Content-Encoding': 'gzip', 'Content-Type': 'application/x-test'}) resp = self.object_controller.POST(req) self.assertEquals(resp.status_int, 202) @@ -80,7 +81,24 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/sda1/p/a/c/o') resp = self.object_controller.GET(req) self.assert_("X-Object-Meta-1" not in resp.headers and \ - "X-Object-Meta-3" in resp.headers) + "X-Object-Meta-Two" not in resp.headers and \ + "X-Object-Meta-3" in resp.headers and \ + "X-Object-Meta-4" in resp.headers and \ + "Content-Encoding" in resp.headers) + self.assertEquals(resp.headers['Content-Type'], 'application/x-test') + + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': timestamp, + 'Content-Type': 'application/x-test'}) + resp = self.object_controller.POST(req) + self.assertEquals(resp.status_int, 202) + req = Request.blank('/sda1/p/a/c/o') + resp = self.object_controller.GET(req) + self.assert_("X-Object-Meta-3" not in resp.headers and \ + "X-Object-Meta-4" not in resp.headers and \ + "Content-Encoding" not in resp.headers) self.assertEquals(resp.headers['Content-Type'], 'application/x-test') def test_POST_not_exist(self): @@ -988,9 +1006,9 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.headers['content-encoding'], 'gzip') req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) + self.assertEquals(resp.headers['content-encoding'], 'gzip') resp = self.object_controller.HEAD(req) self.assertEquals(resp.status_int, 200) - self.assertEquals(resp.headers['content-encoding'], 'gzip') def test_manifest_header(self): timestamp = normalize_timestamp(time()) From 68566d38cfc39dab3e56c781c163f4d22490d1d2 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Tue, 22 Mar 2011 18:25:16 -0500 Subject: [PATCH 07/27] now can delete content-encoding if it was set at object creation --- swift/obj/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/obj/server.py b/swift/obj/server.py index 0914371a33..ee804fa98e 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -137,7 +137,7 @@ class DiskFile(object): if self.meta_file: with open(self.meta_file) as mfp: for key in self.metadata.keys(): - if key.lower() not in ('content-type', 'content-encoding', + if key.lower() not in ('content-type', 'deleted', 'content-length', 'etag'): del self.metadata[key] self.metadata.update(read_metadata(mfp)) From 88ad83767bdae92d7208ad9f29dc3e2938ed514c Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Tue, 22 Mar 2011 20:05:44 -0500 Subject: [PATCH 08/27] objects can now have arbitrary headers set in metadata that will be served back when they are fetched --- etc/object-server.conf-sample | 4 ++++ swift/obj/server.py | 28 +++++++++++++++++++--------- test/unit/obj/test_server.py | 11 +++++++++++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index ea9c22fac3..9d57f8f02d 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -30,6 +30,10 @@ use = egg:swift#object # slow = 1 # on PUTs, sync data every n MB # mb_per_sync = 512 +# Comma separated list of headers that can be set in metadata on an object. +# This list is in addition to X-Object-Meta-* headers and cannot include +# Content-Type, etag, Content-Length, or deleted +# allowed_headers = Content-Encoding [object-replicator] # You can override the default log routing for this app here (don't use set!): diff --git a/swift/obj/server.py b/swift/obj/server.py index ee804fa98e..f0fb302687 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -52,6 +52,8 @@ PICKLE_PROTOCOL = 2 METADATA_KEY = 'user.swift.metadata' MAX_OBJECT_NAME_LENGTH = 1024 KEEP_CACHE_SIZE = (5 * 1024 * 1024) +# keep these lower-case +DISALLOWED_HEADERS = set('content-length content-type deleted etag'.split()) def read_metadata(fd): @@ -137,8 +139,7 @@ class DiskFile(object): if self.meta_file: with open(self.meta_file) as mfp: for key in self.metadata.keys(): - if key.lower() not in ('content-type', - 'deleted', 'content-length', 'etag'): + if key.lower() not in DISALLOWED_HEADERS: del self.metadata[key] self.metadata.update(read_metadata(mfp)) @@ -278,6 +279,10 @@ class ObjectController(object): self.max_upload_time = int(conf.get('max_upload_time', 86400)) self.slow = int(conf.get('slow', 0)) self.bytes_per_sync = int(conf.get('mb_per_sync', 512)) * 1024 * 1024 + default_allowed_headers = 'content-encoding' + self.allowed_headers = set(i.strip().lower() for i in \ + conf.get('allowed_headers', \ + default_allowed_headers).split(',') if i.strip()) def container_update(self, op, account, container, obj, headers_in, headers_out, objdevice): @@ -353,9 +358,11 @@ class ObjectController(object): metadata = {'X-Timestamp': request.headers['x-timestamp']} metadata.update(val for val in request.headers.iteritems() if val[0].lower().startswith('x-object-meta-')) - if 'content-encoding' in request.headers: - metadata['Content-Encoding'] = \ - request.headers['Content-Encoding'] + for header_key in self.allowed_headers: + if header_key in request.headers: + header_caps = \ + header_key.replace('-', ' ').title().replace(' ', '-') + metadata[header_caps] = request.headers[header_key] with file.mkstemp() as (fd, tmppath): file.put(fd, tmppath, metadata, extension='.meta') return response_class(request=request) @@ -420,9 +427,11 @@ class ObjectController(object): metadata.update(val for val in request.headers.iteritems() if val[0].lower().startswith('x-object-meta-') and len(val[0]) > 14) - if 'content-encoding' in request.headers: - metadata['Content-Encoding'] = \ - request.headers['Content-Encoding'] + for header_key in self.allowed_headers: + if header_key in request.headers: + header_caps = \ + header_key.replace('-', ' ').title().replace(' ', '-') + metadata[header_caps] = request.headers[header_key] file.put(fd, tmppath, metadata) file.unlinkold(metadata['X-Timestamp']) self.container_update('PUT', account, container, obj, request.headers, @@ -487,7 +496,8 @@ class ObjectController(object): request=request, conditional_response=True) for key, value in file.metadata.iteritems(): if key == 'X-Object-Manifest' or \ - key.lower().startswith('x-object-meta-'): + key.lower().startswith('x-object-meta-') or \ + key.lower() in self.allowed_headers: response.headers[key] = value response.etag = file.metadata['ETag'] response.last_modified = float(file.metadata['X-Timestamp']) diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index d91ae85cf2..048d1a60bb 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -57,10 +57,14 @@ class TestObjectController(unittest.TestCase): def test_POST_update_meta(self): """ Test swift.object_server.ObjectController.POST """ + test_headers = 'content-encoding foo bar'.split() + self.object_controller.allowed_headers = set(test_headers) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test', + 'Foo': 'fooheader', + 'Baz': 'bazheader', 'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'}) req.body = 'VERIFY' @@ -74,6 +78,8 @@ class TestObjectController(unittest.TestCase): 'X-Object-Meta-3': 'Three', 'X-Object-Meta-4': 'Four', 'Content-Encoding': 'gzip', + 'Foo': 'fooheader', + 'Bar': 'barheader', 'Content-Type': 'application/x-test'}) resp = self.object_controller.POST(req) self.assertEquals(resp.status_int, 202) @@ -84,6 +90,9 @@ class TestObjectController(unittest.TestCase): "X-Object-Meta-Two" not in resp.headers and \ "X-Object-Meta-3" in resp.headers and \ "X-Object-Meta-4" in resp.headers and \ + "Foo" in resp.headers and \ + "Bar" in resp.headers and \ + "Baz" not in resp.headers and \ "Content-Encoding" in resp.headers) self.assertEquals(resp.headers['Content-Type'], 'application/x-test') @@ -98,6 +107,8 @@ class TestObjectController(unittest.TestCase): resp = self.object_controller.GET(req) self.assert_("X-Object-Meta-3" not in resp.headers and \ "X-Object-Meta-4" not in resp.headers and \ + "Foo" not in resp.headers and \ + "Bar" not in resp.headers and \ "Content-Encoding" not in resp.headers) self.assertEquals(resp.headers['Content-Type'], 'application/x-test') From 07c07eba7bf733535aafc1380c5e1f332ffd4d93 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Tue, 22 Mar 2011 22:41:39 -0500 Subject: [PATCH 09/27] simplified header key lookup --- swift/obj/server.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/swift/obj/server.py b/swift/obj/server.py index f0fb302687..d8a2bf19a0 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -360,8 +360,7 @@ class ObjectController(object): if val[0].lower().startswith('x-object-meta-')) for header_key in self.allowed_headers: if header_key in request.headers: - header_caps = \ - header_key.replace('-', ' ').title().replace(' ', '-') + header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] with file.mkstemp() as (fd, tmppath): file.put(fd, tmppath, metadata, extension='.meta') @@ -429,8 +428,7 @@ class ObjectController(object): len(val[0]) > 14) for header_key in self.allowed_headers: if header_key in request.headers: - header_caps = \ - header_key.replace('-', ' ').title().replace(' ', '-') + header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] file.put(fd, tmppath, metadata) file.unlinkold(metadata['X-Timestamp']) From 0374803eb89228b6666243cc12031aad1cda574b Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Tue, 22 Mar 2011 22:54:16 -0500 Subject: [PATCH 10/27] explicitly strip disallowed headers from the config --- swift/obj/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/swift/obj/server.py b/swift/obj/server.py index d8a2bf19a0..89cc1db428 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -282,7 +282,8 @@ class ObjectController(object): default_allowed_headers = 'content-encoding' self.allowed_headers = set(i.strip().lower() for i in \ conf.get('allowed_headers', \ - default_allowed_headers).split(',') if i.strip()) + default_allowed_headers).split(',') if i.strip() and \ + i.strip().lower() not in DISALLOWED_HEADERS) def container_update(self, op, account, container, obj, headers_in, headers_out, objdevice): From d872d944112d87c9f5a35253fcaf1862b366a094 Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 24 Mar 2011 03:37:07 +0000 Subject: [PATCH 11/27] Update from feedback; docs --- doc/source/misc.rst | 6 + etc/proxy-server.conf-sample | 7 + swift/common/middleware/staticweb.py | 418 ++++++++++++++++++++------- swift/common/utils.py | 12 + 4 files changed, 334 insertions(+), 109 deletions(-) diff --git a/doc/source/misc.rst b/doc/source/misc.rst index 5fae5f9a4c..bb856d2fc4 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -128,3 +128,9 @@ Swift3 :members: :show-inheritance: +StaticWeb +========= + +.. automodule:: swift.common.middleware.staticweb + :members: + :show-inheritance: diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 457aa42cf3..07f9d496f3 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -150,3 +150,10 @@ use = egg:swift#cname_lookup # set log_headers = False # storage_domain = example.com # lookup_depth = 1 + +# Note: Put staticweb just after your auth filter(s) in the pipeline +[filter:staticweb] +use = egg:swift#staticweb +# Seconds to cache container x-container-meta-index, x-container-meta-error, +# and x-container-listing-css header values. +# cache_timeout = 300 diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index 319d6a28f4..10311b7d6a 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -13,95 +13,209 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +This StaticWeb WSGI middleware will serve container data as a static web site +with index file and error file resolution and optional file listings. This mode +is normally only active for anonymous requests. If you want to use it with +authenticated requests, set the ``X-Web-Mode: true`` header on the request. + +The ``staticweb`` filter should be added to the pipeline in your +``/etc/swift/proxy-server.conf`` file just after any auth middleware. Also, the +configuration section for the ``staticweb`` middleware itself needs to be +added. For example:: + + [DEFAULT] + ... + + [pipeline:main] + pipeline = healthcheck cache swauth staticweb proxy-server + + ... + + [filter:staticweb] + user = egg:swift#staticweb + # Seconds to cache container x-container-meta-index, + # x-container-meta-error, and x-container-listing-css header values. + # cache_timeout = 300 + +Any publicly readable containers (for example, ``X-Container-Read: .r:*``, see +`acls`_ for more information on this) will be checked for +X-Container-Meta-Index and X-Container-Meta-Error header values:: + + X-Container-Meta-Index + X-Container-Meta-Error + +If X-Container-Meta-Index is set, any files will be served without +having to specify the part. For instance, setting +``X-Container-Meta-Index: index.html`` will be able to serve the object +.../pseudo/path/index.html with just .../pseudo/path or .../pseudo/path/ + +If X-Container-Meta-Error is set, any errors (currently just 401 Unauthorized +and 404 Not Found) will instead serve the .../ +object. For instance, setting ``X-Container-Meta-Error: error.html`` will serve +.../404error.html for requests for paths not found. + +For psuedo paths that have no , this middleware will serve HTML +file listings by default. If you don't want to serve such listings, you can +turn this off via the `acls`_ X-Container-Read setting of ``.rnolisting``. For +example, instead of ``X-Container-Read: .r:*`` you would use +``X-Container-Read: .r:*,.rnolisting`` + +If listings are enabled, the listings can have a custom style sheet by setting +the X-Container-Meta-Listing-CSS header. For instance, setting +``X-Container-Meta-Listing-CSS: listing.css`` will make listings link to the +.../listing.css style sheet. If you "view source" in your browser on a listing +page, you will see the well defined document structure that can be styled. + +Example usage of this middleware via ``st``: + + Make the container publicly readable:: + + st post -r '.r:*' container + + You should be able to get objects and do direct container listings now, + though they'll be in the REST API format. + + Set an index file directive:: + + st post -m 'index:index.html' container + + You should be able to hit paths that have an index.html without needing to + type the index.html part and listings will now be HTML. + + Turn off listings:: + + st post -r '.r:*,.rnolisting' container + + Set an error file:: + + st post -m 'error:error.html' container + + Now 401's should load 401error.html, 404's should load 404error.html, etc. + + Turn listings back on:: + + st post -r '.r:*' container + + Enable a custom listing style sheet:: + + st post -m 'listing-css:listing.css' container +""" + + try: import simplejson as json except ImportError: import json import cgi +import os import urllib from webob import Response, Request from webob.exc import HTTPMovedPermanently, HTTPNotFound, HTTPUnauthorized -from swift.common.utils import split_path, TRUE_VALUES +from swift.common.utils import cache_from_env, get_logger, split_path, \ + TRUE_VALUES -# To use: -# Put the staticweb filter just after the auth filter. -# Make the container publicly readable: -# st post -r '.r:*' container -# You should be able to get objects and do direct container listings -# now, though they'll be in the REST API format. -# Set an index file directive: -# st post -m 'index:index.html' container -# You should be able to hit paths that have an index.html without -# needing to type the index.html part and listings will now be HTML. -# Turn off listings: -# st post -r '.r:*,.rnolisting' container -# Set an error file: -# st post -m 'error:error.html' container -# Now 401's should load s 401error.html, 404's should load -# 404error.html, etc. -# -# This mode is normally only active for anonymous requests. If you -# want to use it with authenticated requests, set the X-Web-Mode: -# true header. -# -# TODO: Make new headers instead of using user metadata. -# TODO: Tests. -# TODO: Docs. -# TODO: get_container_info can be memcached. -# TODO: Blueprint. - class StaticWeb(object): + """ + The Static Web WSGI middleware filter; serves container data as a static + web site. See `staticweb`_ for an overview. + + :param app: The next WSGI application/filter in the paste.deploy pipeline. + :param conf: The filter configuration dict. + """ def __init__(self, app, conf): + #: The next WSGI application/filter in the paste.deploy pipeline. self.app = app + #: The filter configuration dict. self.conf = conf + #: The logger to use with this filter. + self.logger = get_logger(conf, log_route='staticweb') + #: The seconds to cache the x-container-meta-index, + #: x-container-meta-error, and x-container-listing-css headers for a + #: container. + self.cache_timeout = int(conf.get('cache_timeout', 300)) + # Results from the last call to self._start_response. + self._response_status = None + self._response_headers = None + self._response_exc_info = None + # Results from the last call to self._get_container_info. + self._index = self._error = self._listing_css = None - def start_response(self, status, headers, exc_info=None): - self.response_status = status - self.response_headers = headers - self.response_exc_info = exc_info + def _start_response(self, status, headers, exc_info=None): + """ + Saves response info without sending it to the remote client. + Uses the same semantics as the usual WSGI start_response. + """ + self._response_status = status + self._response_headers = headers + self._response_exc_info = exc_info - def error_response(self, response, env, start_response): - if not self.error: - start_response(self.response_status, self.response_headers, - self.response_exc_info) + def _error_response(self, response, env, start_response): + """ + Sends the error response to the remote client, possibly resolving a + custom error response body based on x-container-meta-error. + + :param response: The error response we should default to sending. + :param env: The original request WSGI environment. + :param start_response: The WSGI start_response hook. + """ + if not self._error: + start_response(self._response_status, self._response_headers, + self._response_exc_info) return response - save_response_status = self.response_status - save_response_headers = self.response_headers - save_response_exc_info = self.response_exc_info + save_response_status = self._response_status + save_response_headers = self._response_headers + save_response_exc_info = self._response_exc_info tmp_env = dict(env) - self.strip_ifs(tmp_env) + self._strip_ifs(tmp_env) tmp_env['PATH_INFO'] = '/%s/%s/%s/%s%s' % (self.version, self.account, - self.container, self.get_status_int(), self.error) + self.container, self._get_status_int(), self._error) tmp_env['REQUEST_METHOD'] = 'GET' - resp = self.app(tmp_env, self.start_response) - if self.get_status_int() // 100 == 2: - start_response(self.response_status, self.response_headers, - self.response_exc_info) + resp = self.app(tmp_env, self._start_response) + if self._get_status_int() // 100 == 2: + start_response(save_response_status, self._response_headers, + self._response_exc_info) return resp start_response(save_response_status, save_response_headers, save_response_exc_info) return response - def get_status_int(self): - return int(self.response_status.split(' ', 1)[0]) + def _get_status_int(self): + """ + Returns the HTTP status int from the last called self._start_response + result. + """ + return int(self._response_status.split(' ', 1)[0]) - def get_header(self, headers, name, default_value=None): - for header, value in headers: - if header.lower() == name: - return value - return default_value - - def strip_ifs(self, env): + def _strip_ifs(self, env): + """ Strips any HTTP_IF_* keys from the env dict. """ for key in [k for k in env.keys() if k.startswith('HTTP_IF_')]: del env[key] - def get_container_info(self, env, start_response): - self.index = self.error = None + def _get_container_info(self, env, start_response): + """ + Retrieves x-container-meta-index, x-container-meta-error, and + x-container-meta-listing-css from memcache or from the cluster and + stores the result in memcache and in self._index, self._error, and + self._listing_css. + + :param env: The WSGI environment dict. + :param start_response: The WSGI start_response hook. + """ + self._index = self._error = self._listing_css = None + memcache_client = cache_from_env(env) + if memcache_client: + memcache_key = '/staticweb/%s/%s/%s' % (self.version, self.account, + self.container) + cached_data = memcache_client.get(memcache_key) + if cached_data: + self._index, self._error, self._listing_css = cached_data + return tmp_env = {'REQUEST_METHOD': 'HEAD', 'HTTP_USER_AGENT': 'StaticWeb'} for name in ('swift.cache', 'HTTP_X_CF_TRANS_ID'): if name in env: @@ -110,127 +224,213 @@ class StaticWeb(object): self.container), environ=tmp_env) resp = req.get_response(self.app) if resp.status_int // 100 == 2: - self.index = resp.headers.get('x-container-meta-index', '').strip() - self.error = resp.headers.get('x-container-meta-error', '').strip() + self._index = \ + resp.headers.get('x-container-meta-index', '').strip() + self._listing_css = \ + resp.headers.get('x-container-meta-listing-css', '').strip() + self._error = \ + resp.headers.get('x-container-meta-error', '').strip() + if memcache_client: + memcache_client.set(memcache_key, + (self._index, self._error, self._listing_css), + timeout=self.cache_timeout) - def listing(self, env, start_response, prefix=None): + def _listing(self, env, start_response, prefix=None): + """ + Sends an HTML object listing to the remote client. + + :param env: The original WSGI environment dict. + :param start_response: The original WSGI start_response hook. + :param prefix: Any prefix desired for the container listing. + """ tmp_env = dict(env) - self.strip_ifs(tmp_env) + self._strip_ifs(tmp_env) tmp_env['REQUEST_METHOD'] = 'GET' tmp_env['PATH_INFO'] = \ '/%s/%s/%s' % (self.version, self.account, self.container) tmp_env['QUERY_STRING'] = 'delimiter=/&format=json' if prefix: tmp_env['QUERY_STRING'] += '&prefix=%s' % urllib.quote(prefix) - resp = self.app(tmp_env, self.start_response) - if self.get_status_int() // 100 != 2: - return self.error_response(resp, env, start_response) + resp = self.app(tmp_env, self._start_response) + if self._get_status_int() // 100 != 2: + return self._error_response(resp, env, start_response) listing = json.loads(''.join(resp)) if not listing: - resp = HTTPNotFound()(env, self.start_response) - return self.error_response(resp, env, start_response) + resp = HTTPNotFound()(env, self._start_response) + return self._error_response(resp, env, start_response) headers = {'Content-Type': 'text/html'} - body = 'Listing of%s' \ - '

Listing of %s

\n' % \ - (cgi.escape(env['PATH_INFO']), cgi.escape(env['PATH_INFO'])) + body = '\n' \ + '\n' \ + ' \n' \ + ' Listing of %s\n' % \ + cgi.escape(env['PATH_INFO']) + if self._listing_css: + body += ' \n' % \ + (self.version, self.account, self.container, + urllib.quote(self._listing_css)) + else: + body += ' \n' + body += ' \n' \ + ' \n' \ + '

Listing of %s

\n' \ + ' \n' \ + ' \n' \ + ' \n' \ + ' \n' \ + ' \n' \ + ' \n' % \ + cgi.escape(env['PATH_INFO']) if prefix: - body += '../
' + body += ' \n' \ + ' \n' \ + ' \n' \ + ' \n' \ + ' \n' for item in listing: if 'subdir' in item: subdir = item['subdir'] if prefix: subdir = subdir[len(prefix):] - body += '%s
' % \ + body += ' \n' \ + ' \n' \ + ' \n' \ + ' \n' \ + ' \n' % \ (urllib.quote(subdir), cgi.escape(subdir)) for item in listing: if 'name' in item: name = item['name'] if prefix: name = name[len(prefix):] - body += '%s
' % \ - (urllib.quote(name), cgi.escape(name)) - body += '

\n' + body += ' \n' \ + ' \n' \ + ' \n' \ + ' \n' \ + ' \n' % \ + (' '.join('type-' + cgi.escape(t.lower(), quote=True) + for t in item['content_type'].split('/')), + urllib.quote(name), cgi.escape(name), + self.human_readable(item['bytes']), + cgi.escape(item['last_modified']).split('.')[0]. + replace('T', ' ')) + body += '
NameSizeDate
../  
%s  
%s%s%s
\n' \ + ' \n' \ + '\n' return Response(headers=headers, body=body)(env, start_response) - def handle_container(self, env, start_response): - self.get_container_info(env, start_response) - if not self.index: + def _handle_container(self, env, start_response): + """ + Handles a possible static web request for a container. + + :param env: The original WSGI environment dict. + :param start_response: The original WSGI start_response hook. + """ + self._get_container_info(env, start_response) + if not self._index: return self.app(env, start_response) if env['PATH_INFO'][-1] != '/': return HTTPMovedPermanently( - location=env['PATH_INFO'] + '/')(env, start_response) + location=(env['PATH_INFO'] + '/'))(env, start_response) tmp_env = dict(env) - tmp_env['PATH_INFO'] += self.index - resp = self.app(tmp_env, self.start_response) - status_int = self.get_status_int() + tmp_env['PATH_INFO'] += self._index + resp = self.app(tmp_env, self._start_response) + status_int = self._get_status_int() if status_int == 404: - return self.listing(env, start_response) - elif self.get_status_int() // 100 not in (2, 3): - return self.error_response(resp, env, start_response) - start_response(self.response_status, self.response_headers, - self.response_exc_info) + return self._listing(env, start_response) + elif self._get_status_int() // 100 not in (2, 3): + return self._error_response(resp, env, start_response) + start_response(self._response_status, self._response_headers, + self._response_exc_info) return resp - def handle_object(self, env, start_response): + def _handle_object(self, env, start_response): + """ + Handles a possible static web request for an object. This object could + resolve into an index or listing request. + + :param env: The original WSGI environment dict. + :param start_response: The original WSGI start_response hook. + """ tmp_env = dict(env) - resp = self.app(tmp_env, self.start_response) - status_int = self.get_status_int() + resp = self.app(tmp_env, self._start_response) + status_int = self._get_status_int() if status_int // 100 in (2, 3): - start_response(self.response_status, self.response_headers, - self.response_exc_info) + start_response(self._response_status, self._response_headers, + self._response_exc_info) return resp if status_int != 404: - return self.error_response(resp, env, start_response) - self.get_container_info(env, start_response) - if not self.index: + return self._error_response(resp, env, start_response) + self._get_container_info(env, start_response) + if not self._index: return self.app(env, start_response) tmp_env = dict(env) if tmp_env['PATH_INFO'][-1] != '/': tmp_env['PATH_INFO'] += '/' - tmp_env['PATH_INFO'] += self.index - resp = self.app(tmp_env, self.start_response) - status_int = self.get_status_int() + tmp_env['PATH_INFO'] += self._index + resp = self.app(tmp_env, self._start_response) + status_int = self._get_status_int() if status_int // 100 in (2, 3): if env['PATH_INFO'][-1] != '/': return HTTPMovedPermanently( location=env['PATH_INFO'] + '/')(env, start_response) - start_response(self.response_status, self.response_headers, - self.response_exc_info) + start_response(self._response_status, self._response_headers, + self._response_exc_info) return resp elif status_int == 404: if env['PATH_INFO'][-1] != '/': tmp_env = dict(env) - self.strip_ifs(tmp_env) + self._strip_ifs(tmp_env) tmp_env['REQUEST_METHOD'] = 'GET' tmp_env['PATH_INFO'] = '/%s/%s/%s' % (self.version, self.account, self.container) tmp_env['QUERY_STRING'] = 'limit=1&format=json&delimiter' \ '=/&limit=1&prefix=%s' % urllib.quote(self.obj + '/') - resp = self.app(tmp_env, self.start_response) - if self.get_status_int() // 100 != 2 or \ + resp = self.app(tmp_env, self._start_response) + if self._get_status_int() // 100 != 2 or \ not json.loads(''.join(resp)): - resp = HTTPNotFound()(env, self.start_response) - return self.error_response(resp, env, start_response) + resp = HTTPNotFound()(env, self._start_response) + return self._error_response(resp, env, start_response) return HTTPMovedPermanently(location=env['PATH_INFO'] + '/')(env, start_response) - return self.listing(env, start_response, self.obj) + return self._listing(env, start_response, self.obj) def __call__(self, env, start_response): + """ + Main hook into the WSGI paste.deploy filter/app pipeline. + + :param env: The WSGI environment dict. + :param start_response: The WSGI start_response hook. + """ + (self.version, self.account, self.container, self.obj) = \ + split_path(env['PATH_INFO'], 2, 4, True) + memcache_client = cache_from_env(env) + if memcache_client: + if env['REQUEST_METHOD'] in ('PUT', 'POST'): + if not self.obj and self.container: + memcache_key = '/staticweb/%s/%s/%s' % \ + (self.version, self.account, self.container) + memcache_client.delete(memcache_key) + return self.app(env, start_response) if env['REQUEST_METHOD'] not in ('HEAD', 'GET') or \ (env.get('REMOTE_USER') and not env.get('HTTP_X_WEB_MODE', '') in TRUE_VALUES): return self.app(env, start_response) - (self.version, self.account, self.container, self.obj) = \ - split_path(env['PATH_INFO'], 2, 4, True) if self.obj: - return self.handle_object(env, start_response) + return self._handle_object(env, start_response) elif self.container: - return self.handle_container(env, start_response) + return self._handle_container(env, start_response) return self.app(env, start_response) def filter_factory(global_conf, **local_conf): - """Returns a WSGI filter app for use with paste.deploy.""" + """ Returns a Static Web WSGI filter for use with paste.deploy. """ conf = global_conf.copy() conf.update(local_conf) diff --git a/swift/common/utils.py b/swift/common/utils.py index c867a55821..416a482817 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -17,6 +17,7 @@ import errno import fcntl +import math import os import pwd import signal @@ -969,3 +970,14 @@ def urlparse(url): :param url: URL to parse. """ return ModifiedParseResult(*stdlib_urlparse(url)) + + +def human_readable(self, n): + """ + Returns the number in a human readable format; for example 1000000 = "1m". + Idea from: http://stackoverflow.com/questions/3154460/ + """ + millnames = ['', 'k', 'm', 'g', 't', 'p', 'e'] + millidx = max(0, min(len(millnames) - 1, + int(math.floor(math.log10(abs(n)) / 3.0)))) + return '%.0f%s' % (n / 10 ** (3 * millidx), millnames[millidx]) From b09b5e64e1843c5155b91e8c053e3f3270b3cdd6 Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 24 Mar 2011 07:46:02 +0000 Subject: [PATCH 12/27] Tests; bug fixes --- swift/common/middleware/staticweb.py | 20 +- swift/common/middleware/swauth.py | 2 +- swift/common/utils.py | 21 +- test/unit/common/middleware/test_staticweb.py | 488 ++++++++++++++++++ test/unit/common/middleware/test_swauth.py | 25 +- test/unit/common/test_utils.py | 23 + 6 files changed, 551 insertions(+), 28 deletions(-) create mode 100644 test/unit/common/middleware/test_staticweb.py diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index 10311b7d6a..c58858793a 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -82,7 +82,7 @@ Example usage of this middleware via ``st``: You should be able to hit paths that have an index.html without needing to type the index.html part and listings will now be HTML. - + Turn off listings:: st post -r '.r:*,.rnolisting' container @@ -109,13 +109,12 @@ except ImportError: import json import cgi -import os import urllib from webob import Response, Request -from webob.exc import HTTPMovedPermanently, HTTPNotFound, HTTPUnauthorized +from webob.exc import HTTPMovedPermanently, HTTPNotFound -from swift.common.utils import cache_from_env, get_logger, split_path, \ +from swift.common.utils import cache_from_env, human_readable, split_path, \ TRUE_VALUES @@ -133,8 +132,6 @@ class StaticWeb(object): self.app = app #: The filter configuration dict. self.conf = conf - #: The logger to use with this filter. - self.logger = get_logger(conf, log_route='staticweb') #: The seconds to cache the x-container-meta-index, #: x-container-meta-error, and x-container-listing-css headers for a #: container. @@ -317,7 +314,7 @@ class StaticWeb(object): (' '.join('type-' + cgi.escape(t.lower(), quote=True) for t in item['content_type'].split('/')), urllib.quote(name), cgi.escape(name), - self.human_readable(item['bytes']), + human_readable(item['bytes']), cgi.escape(item['last_modified']).split('.')[0]. replace('T', ' ')) body += ' \n' \ @@ -408,8 +405,11 @@ class StaticWeb(object): :param env: The WSGI environment dict. :param start_response: The WSGI start_response hook. """ - (self.version, self.account, self.container, self.obj) = \ - split_path(env['PATH_INFO'], 2, 4, True) + try: + (self.version, self.account, self.container, self.obj) = \ + split_path(env['PATH_INFO'], 2, 4, True) + except ValueError: + return self.app(env, start_response) memcache_client = cache_from_env(env) if memcache_client: if env['REQUEST_METHOD'] in ('PUT', 'POST'): @@ -420,7 +420,7 @@ class StaticWeb(object): return self.app(env, start_response) if env['REQUEST_METHOD'] not in ('HEAD', 'GET') or \ (env.get('REMOTE_USER') and - not env.get('HTTP_X_WEB_MODE', '') in TRUE_VALUES): + env.get('HTTP_X_WEB_MODE', '') not in TRUE_VALUES): return self.app(env, start_response) if self.obj: return self._handle_object(env, start_response) diff --git a/swift/common/middleware/swauth.py b/swift/common/middleware/swauth.py index 86cdcac07c..59c8fd66de 100644 --- a/swift/common/middleware/swauth.py +++ b/swift/common/middleware/swauth.py @@ -278,7 +278,7 @@ class Swauth(object): referrers, groups = parse_acl(getattr(req, 'acl', None)) if referrer_allowed(req.referer, referrers): if not obj and '.rnolisting' in groups: - return HTTPUnauthorized(request=req) + return self.denied_response(req) return None if not req.remote_user: return self.denied_response(req) diff --git a/swift/common/utils.py b/swift/common/utils.py index 416a482817..41874e8073 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -17,7 +17,6 @@ import errno import fcntl -import math import os import pwd import signal @@ -73,7 +72,7 @@ if hash_conf.read('/etc/swift/swift.conf'): pass # Used when reading config values -TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes', 'on', 'On')) +TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes', 'on', 'On', 't', 'y')) def validate_configuration(): @@ -972,12 +971,16 @@ def urlparse(url): return ModifiedParseResult(*stdlib_urlparse(url)) -def human_readable(self, n): +def human_readable(value): """ - Returns the number in a human readable format; for example 1000000 = "1m". - Idea from: http://stackoverflow.com/questions/3154460/ + Returns the number in a human readable format; for example 1048576 = "1Mi". """ - millnames = ['', 'k', 'm', 'g', 't', 'p', 'e'] - millidx = max(0, min(len(millnames) - 1, - int(math.floor(math.log10(abs(n)) / 3.0)))) - return '%.0f%s' % (n / 10 ** (3 * millidx), millnames[millidx]) + value = float(value) + index = -1 + suffixes = 'KMGTPEZY' + while value >= 1024 and index + 1 < len(suffixes): + index += 1 + value = round(value / 1024) + if index == -1: + return '%d' % value + return '%d%si' % (round(value), suffixes[index]) diff --git a/test/unit/common/middleware/test_staticweb.py b/test/unit/common/middleware/test_staticweb.py new file mode 100644 index 0000000000..61a4107249 --- /dev/null +++ b/test/unit/common/middleware/test_staticweb.py @@ -0,0 +1,488 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + import simplejson as json +except ImportError: + import json +import unittest +from contextlib import contextmanager + +from webob import Request, Response + +from swift.common.middleware import staticweb + + +class FakeMemcache(object): + + def __init__(self): + self.store = {} + + def get(self, key): + return self.store.get(key) + + def set(self, key, value, timeout=0): + self.store[key] = value + return True + + def incr(self, key, timeout=0): + self.store[key] = self.store.setdefault(key, 0) + 1 + return self.store[key] + + @contextmanager + def soft_lock(self, key, timeout=0, retries=5): + yield True + + def delete(self, key): + try: + del self.store[key] + except Exception: + pass + return True + + +class FakeApp(object): + + def __init__(self, status_headers_body_iter=None): + self.get_c4_called = False + + def __call__(self, env, start_response): + if env['PATH_INFO'] == '/': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1': + return Response( + status='412 Precondition Failed')(env, start_response) + elif env['PATH_INFO'] == '/v1/a': + return Response(status='401 Unauthorized')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c1': + return Response(status='401 Unauthorized')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c2': + return self.listing(env, start_response, + {'x-container-read': '.r:*'}) + elif env['PATH_INFO'] == '/v1/a/c2/one.txt': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3': + return self.listing(env, start_response, + {'x-container-read': '.r:*', + 'x-container-meta-index': 'index.html'}) + elif env['PATH_INFO'] == '/v1/a/c3/index.html': + return Response(status='200 Ok', body=''' + + +

Test main index.html file.

+

Visit subdir.

+

Don't visit subdir2 because it doesn't really + exist.

+

Visit subdir3.

+

Visit subdir3/subsubdir.

+ + + ''')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3/subdir': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3/subdir/': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3/subdir/index.html': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3/subdir3/subsubdir': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3/subdir3/subsubdir/': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3/subdir3/subsubdir/index.html': + return Response(status='200 Ok', body='index file')(env, + start_response) + elif env['PATH_INFO'] == '/v1/a/c3/subdirx/': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3/subdirx/index.html': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3/subdiry/': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3/subdiry/index.html': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3/subdirz': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3/subdirz/index.html': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3/unknown': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c3/unknown/index.html': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c4': + self.get_c4_called = True + return self.listing(env, start_response, + {'x-container-read': '.r:*', + 'x-container-meta-index': 'index.html', + 'x-container-meta-error': 'error.html', + 'x-container-meta-listing-css': 'listing.css'}) + elif env['PATH_INFO'] == '/v1/a/c4/one.txt': + return Response(status='200 Ok', body='1')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c4/two.txt': + return Response(status='503 Service Unavailable')(env, + start_response) + elif env['PATH_INFO'] == '/v1/a/c4/index.html': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c4/subdir/': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c4/subdir/index.html': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c4/unknown': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c4/unknown/index.html': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c4/404error.html': + return Response(status='200 Ok', body=''' + + +

Chrome's 404 fancy-page sucks.

+ + + '''.strip())(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c5': + return self.listing(env, start_response, + {'x-container-read': '.r:*', + 'x-container-meta-index': 'index.html', + 'x-container-meta-error': 'error.html'}) + elif env['PATH_INFO'] == '/v1/a/c5/index.html': + return Response(status='503 Service Unavailable')(env, + start_response) + elif env['PATH_INFO'] == '/v1/a/c5/503error.html': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c5/unknown': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c5/unknown/index.html': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c5/404error.html': + return Response(status='404 Not Found')(env, start_response) + else: + raise Exception('Unknown path %r' % env['PATH_INFO']) + + def listing(self, env, start_response, headers): + if env['PATH_INFO'] in ('/v1/a/c3', '/v1/a/c4') and \ + env['QUERY_STRING'] == 'delimiter=/&format=json&prefix=subdir/': + headers.update({'X-Container-Object-Count': '11', + 'X-Container-Bytes-Used': '73741', + 'X-Container-Read': '.r:*', + 'Content-Type': 'application/json; charset=utf8'}) + body = ''' + [{"name":"subdir/1.txt", + "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, + "content_type":"text/plain", + "last_modified":"2011-03-24T04:27:52.709100"}, + {"name":"subdir/2.txt", + "hash":"c85c1dcd19cf5cbac84e6043c31bb63e", "bytes":20, + "content_type":"text/plain", + "last_modified":"2011-03-24T04:27:52.734140"}, + {"subdir":"subdir3/subsubdir/"}] + '''.strip() + elif env['PATH_INFO'] == '/v1/a/c3' and env['QUERY_STRING'] == \ + 'delimiter=/&format=json&prefix=subdiry/': + headers.update({'X-Container-Object-Count': '11', + 'X-Container-Bytes-Used': '73741', + 'X-Container-Read': '.r:*', + 'Content-Type': 'application/json; charset=utf8'}) + body = '[]' + elif env['PATH_INFO'] == '/v1/a/c3' and env['QUERY_STRING'] == \ + 'limit=1&format=json&delimiter=/&limit=1&prefix=subdirz/': + headers.update({'X-Container-Object-Count': '11', + 'X-Container-Bytes-Used': '73741', + 'X-Container-Read': '.r:*', + 'Content-Type': 'application/json; charset=utf8'}) + body = ''' + [{"name":"subdirz/1.txt", + "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, + "content_type":"text/plain", + "last_modified":"2011-03-24T04:27:52.709100"}] + '''.strip() + elif 'prefix=' in env['QUERY_STRING']: + return Response(status='204 No Content') + elif 'format=json' in env['QUERY_STRING']: + headers.update({'X-Container-Object-Count': '11', + 'X-Container-Bytes-Used': '73741', + 'Content-Type': 'application/json; charset=utf8'}) + body = ''' + [{"name":"401error.html", + "hash":"893f8d80692a4d3875b45be8f152ad18", "bytes":110, + "content_type":"text/html", + "last_modified":"2011-03-24T04:27:52.713710"}, + {"name":"404error.html", + "hash":"62dcec9c34ed2b347d94e6ca707aff8c", "bytes":130, + "content_type":"text/html", + "last_modified":"2011-03-24T04:27:52.720850"}, + {"name":"index.html", + "hash":"8b469f2ca117668a5131fe9ee0815421", "bytes":347, + "content_type":"text/html", + "last_modified":"2011-03-24T04:27:52.683590"}, + {"name":"listing.css", + "hash":"7eab5d169f3fcd06a08c130fa10c5236", "bytes":17, + "content_type":"text/css", + "last_modified":"2011-03-24T04:27:52.721610"}, + {"name":"one.txt", "hash":"73f1dd69bacbf0847cc9cffa3c6b23a1", + "bytes":22, "content_type":"text/plain", + "last_modified":"2011-03-24T04:27:52.722270"}, + {"name":"subdir/1.txt", + "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, + "content_type":"text/plain", + "last_modified":"2011-03-24T04:27:52.709100"}, + {"name":"subdir/2.txt", + "hash":"c85c1dcd19cf5cbac84e6043c31bb63e", "bytes":20, + "content_type":"text/plain", + "last_modified":"2011-03-24T04:27:52.734140"}, + {"name":"subdir/omgomg.txt", + "hash":"7337d028c093130898d937c319cc9865", "bytes":72981, + "content_type":"text/plain", + "last_modified":"2011-03-24T04:27:52.735460"}, + {"name":"subdir2", "hash":"d41d8cd98f00b204e9800998ecf8427e", + "bytes":0, "content_type":"text/directory", + "last_modified":"2011-03-24T04:27:52.676690"}, + {"name":"subdir3/subsubdir/index.html", + "hash":"04eea67110f883b1a5c97eb44ccad08c", "bytes":72, + "content_type":"text/html", + "last_modified":"2011-03-24T04:27:52.751260"}, + {"name":"two.txt", "hash":"10abb84c63a5cff379fdfd6385918833", + "bytes":22, "content_type":"text/plain", + "last_modified":"2011-03-24T04:27:52.825110"}] + '''.strip() + else: + headers.update({'X-Container-Object-Count': '11', + 'X-Container-Bytes-Used': '73741', + 'Content-Type': 'text/plain; charset=utf8'}) + body = '\n'.join(['401error.html', '404error.html', 'index.html', + 'listing.css', 'one.txt', 'subdir/1.txt', + 'subdir/2.txt', 'subdir/omgomg.txt', 'subdir2', + 'subdir3/subsubdir/index.html', 'two.txt']) + return Response(status='200 Ok', headers=headers, + body=body)(env, start_response) + + +class TestStaticWeb(unittest.TestCase): + + def setUp(self): + self.test_staticweb = staticweb.filter_factory({})(FakeApp()) + + def test_app_set(self): + app = FakeApp() + sw = staticweb.filter_factory({})(app) + self.assertEquals(sw.app, app) + + def test_conf_set(self): + conf = {'blah': 1} + sw = staticweb.filter_factory(conf)(FakeApp()) + self.assertEquals(sw.conf, conf) + + def test_cache_timeout_unset(self): + sw = staticweb.filter_factory({})(FakeApp()) + self.assertEquals(sw.cache_timeout, 300) + + def test_cache_timeout_set(self): + sw = staticweb.filter_factory({'cache_timeout': '1'})(FakeApp()) + self.assertEquals(sw.cache_timeout, 1) + + def test_root(self): + resp = Request.blank('/').get_response(self.test_staticweb) + self.assertEquals(resp.status_int, 404) + + def test_version(self): + resp = Request.blank('/v1').get_response(self.test_staticweb) + self.assertEquals(resp.status_int, 412) + + def test_account(self): + resp = Request.blank('/v1/a').get_response(self.test_staticweb) + self.assertEquals(resp.status_int, 401) + + def test_container1(self): + resp = Request.blank('/v1/a/c1').get_response(self.test_staticweb) + self.assertEquals(resp.status_int, 401) + + def test_container2(self): + resp = Request.blank('/v1/a/c2').get_response(self.test_staticweb) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_type, 'text/plain') + self.assertEquals(len(resp.body.split('\n')), + int(resp.headers['x-container-object-count'])) + + def test_container2onetxt(self): + resp = Request.blank( + '/v1/a/c2/one.txt').get_response(self.test_staticweb) + self.assertEquals(resp.status_int, 404) + + def test_container2json(self): + resp = Request.blank( + '/v1/a/c2?format=json').get_response(self.test_staticweb) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.content_type, 'application/json') + self.assertEquals(len(json.loads(resp.body)), + int(resp.headers['x-container-object-count'])) + + def test_container3(self): + resp = Request.blank('/v1/a/c3').get_response(self.test_staticweb) + self.assertEquals(resp.status_int, 301) + self.assertEquals(resp.headers['location'], + 'http://localhost/v1/a/c3/') + + def test_container3indexhtml(self): + resp = Request.blank('/v1/a/c3/').get_response(self.test_staticweb) + self.assertEquals(resp.status_int, 200) + self.assert_('Test main index.html file.' in resp.body) + + def test_container3subdir(self): + resp = Request.blank( + '/v1/a/c3/subdir').get_response(self.test_staticweb) + self.assertEquals(resp.status_int, 301) + + def test_container3subsubdir(self): + resp = Request.blank( + '/v1/a/c3/subdir3/subsubdir').get_response(self.test_staticweb) + self.assertEquals(resp.status_int, 301) + + def test_container3subsubdircontents(self): + resp = Request.blank( + '/v1/a/c3/subdir3/subsubdir/').get_response(self.test_staticweb) + self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.body, 'index file') + + def test_container3subdir(self): + resp = Request.blank( + '/v1/a/c3/subdir/').get_response(self.test_staticweb) + self.assertEquals(resp.status_int, 200) + self.assert_('Listing of /v1/a/c3/subdir/' in resp.body) + self.assert_('' in resp.body) + self.assert_('' not in resp.body) + self.assert_(' Date: Thu, 24 Mar 2011 11:43:12 -0500 Subject: [PATCH 13/27] fixed location of a test --- test/unit/obj/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 048d1a60bb..904eb93c79 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -1017,9 +1017,9 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.headers['content-encoding'], 'gzip') req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) - self.assertEquals(resp.headers['content-encoding'], 'gzip') resp = self.object_controller.HEAD(req) self.assertEquals(resp.status_int, 200) + self.assertEquals(resp.headers['content-encoding'], 'gzip') def test_manifest_header(self): timestamp = normalize_timestamp(time()) From 0c5aacb42467eb5e34b2c29436ebceb28605e41f Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Thu, 24 Mar 2011 13:03:49 -0500 Subject: [PATCH 14/27] added default support for content-disposition and allows x-object-manifest to be manipulated like any other object metadata header --- etc/object-server.conf-sample | 2 +- swift/obj/server.py | 9 ++--- test/unit/obj/test_server.py | 66 +++++++++++++++++++++++++++++------ 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index 9d57f8f02d..8831048e43 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -33,7 +33,7 @@ use = egg:swift#object # Comma separated list of headers that can be set in metadata on an object. # This list is in addition to X-Object-Meta-* headers and cannot include # Content-Type, etag, Content-Length, or deleted -# allowed_headers = Content-Encoding +# allowed_headers = Content-Encoding, Content-Disposition, X-Object-Manifest [object-replicator] # You can override the default log routing for this app here (don't use set!): diff --git a/swift/obj/server.py b/swift/obj/server.py index 89cc1db428..0d5207e788 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -279,7 +279,8 @@ class ObjectController(object): self.max_upload_time = int(conf.get('max_upload_time', 86400)) self.slow = int(conf.get('slow', 0)) self.bytes_per_sync = int(conf.get('mb_per_sync', 512)) * 1024 * 1024 - default_allowed_headers = 'content-encoding' + default_allowed_headers = 'content-encoding, x-object-manifest, ' \ + 'content-disposition' self.allowed_headers = set(i.strip().lower() for i in \ conf.get('allowed_headers', \ default_allowed_headers).split(',') if i.strip() and \ @@ -421,9 +422,6 @@ class ObjectController(object): 'ETag': etag, 'Content-Length': str(os.fstat(fd).st_size), } - if 'x-object-manifest' in request.headers: - metadata['X-Object-Manifest'] = \ - request.headers['x-object-manifest'] metadata.update(val for val in request.headers.iteritems() if val[0].lower().startswith('x-object-meta-') and len(val[0]) > 14) @@ -494,8 +492,7 @@ class ObjectController(object): 'application/octet-stream'), app_iter=file, request=request, conditional_response=True) for key, value in file.metadata.iteritems(): - if key == 'X-Object-Manifest' or \ - key.lower().startswith('x-object-meta-') or \ + if key.lower().startswith('x-object-meta-') or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = file.metadata['ETag'] diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 904eb93c79..3b2e8f6a61 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -57,6 +57,7 @@ class TestObjectController(unittest.TestCase): def test_POST_update_meta(self): """ Test swift.object_server.ObjectController.POST """ + original_headers = self.object_controller.allowed_headers test_headers = 'content-encoding foo bar'.split() self.object_controller.allowed_headers = set(test_headers) timestamp = normalize_timestamp(time()) @@ -86,13 +87,13 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/sda1/p/a/c/o') resp = self.object_controller.GET(req) - self.assert_("X-Object-Meta-1" not in resp.headers and \ - "X-Object-Meta-Two" not in resp.headers and \ - "X-Object-Meta-3" in resp.headers and \ - "X-Object-Meta-4" in resp.headers and \ - "Foo" in resp.headers and \ - "Bar" in resp.headers and \ - "Baz" not in resp.headers and \ + self.assert_("X-Object-Meta-1" not in resp.headers and + "X-Object-Meta-Two" not in resp.headers and + "X-Object-Meta-3" in resp.headers and + "X-Object-Meta-4" in resp.headers and + "Foo" in resp.headers and + "Bar" in resp.headers and + "Baz" not in resp.headers and "Content-Encoding" in resp.headers) self.assertEquals(resp.headers['Content-Type'], 'application/x-test') @@ -105,13 +106,56 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 202) req = Request.blank('/sda1/p/a/c/o') resp = self.object_controller.GET(req) - self.assert_("X-Object-Meta-3" not in resp.headers and \ - "X-Object-Meta-4" not in resp.headers and \ - "Foo" not in resp.headers and \ - "Bar" not in resp.headers and \ + self.assert_("X-Object-Meta-3" not in resp.headers and + "X-Object-Meta-4" not in resp.headers and + "Foo" not in resp.headers and + "Bar" not in resp.headers and "Content-Encoding" not in resp.headers) self.assertEquals(resp.headers['Content-Type'], 'application/x-test') + # test defaults + self.object_controller.allowed_headers = original_headers + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Type': 'application/x-test', + 'Foo': 'fooheader', + 'X-Object-Meta-1': 'One', + 'X-Object-Manifest': 'c/bar', + 'Content-Encoding': 'gzip', + 'Content-Disposition': 'bar', + }) + req.body = 'VERIFY' + resp = self.object_controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c/o') + resp = self.object_controller.GET(req) + self.assert_("X-Object-Meta-1" in resp.headers and + "Foo" not in resp.headers and + "Content-Encoding" in resp.headers and + "X-Object-Manifest" in resp.headers and + "Content-Disposition" in resp.headers) + self.assertEquals(resp.headers['Content-Type'], 'application/x-test') + + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': timestamp, + 'X-Object-Meta-3': 'Three', + 'Foo': 'fooheader', + 'Content-Type': 'application/x-test'}) + resp = self.object_controller.POST(req) + self.assertEquals(resp.status_int, 202) + req = Request.blank('/sda1/p/a/c/o') + resp = self.object_controller.GET(req) + self.assert_("X-Object-Meta-1" not in resp.headers and + "Foo" not in resp.headers and + "Content-Encoding" not in resp.headers and + "X-Object-Manifest" not in resp.headers and + "Content-Disposition" not in resp.headers and + "X-Object-Meta-3" in resp.headers) + self.assertEquals(resp.headers['Content-Type'], 'application/x-test') + def test_POST_not_exist(self): timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/fail', From f3d4f7311bf2d0b5f4faef641c7440d558ff7aaf Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 24 Mar 2011 19:22:54 +0000 Subject: [PATCH 15/27] Typo in docs --- swift/common/middleware/staticweb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index c58858793a..248fa4fe72 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -33,7 +33,7 @@ added. For example:: ... [filter:staticweb] - user = egg:swift#staticweb + use = egg:swift#staticweb # Seconds to cache container x-container-meta-index, # x-container-meta-error, and x-container-listing-css header values. # cache_timeout = 300 From 51ce438f7ff112dddbef79e795a54485f29e9c18 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Thu, 24 Mar 2011 15:51:44 -0700 Subject: [PATCH 16/27] changing /usr/bin/python to /usr/bin/env python --- bin/st | 2 +- bin/swauth-add-account | 2 +- bin/swauth-add-user | 2 +- bin/swauth-cleanup-tokens | 2 +- bin/swauth-delete-account | 2 +- bin/swauth-delete-user | 2 +- bin/swauth-list | 2 +- bin/swauth-prep | 2 +- bin/swauth-set-account-service | 2 +- bin/swift-account-audit | 2 +- bin/swift-account-auditor | 2 +- bin/swift-account-reaper | 2 +- bin/swift-account-replicator | 2 +- bin/swift-account-server | 2 +- bin/swift-account-stats-logger | 2 +- bin/swift-bench | 2 +- bin/swift-container-auditor | 2 +- bin/swift-container-replicator | 2 +- bin/swift-container-server | 2 +- bin/swift-container-updater | 2 +- bin/swift-drive-audit | 2 +- bin/swift-get-nodes | 2 +- bin/swift-init | 2 +- bin/swift-log-stats-collector | 2 +- bin/swift-log-uploader | 2 +- bin/swift-object-auditor | 2 +- bin/swift-object-info | 2 +- bin/swift-object-replicator | 2 +- bin/swift-object-server | 2 +- bin/swift-object-updater | 2 +- bin/swift-proxy-server | 2 +- bin/swift-ring-builder | 2 +- bin/swift-stats-populate | 2 +- bin/swift-stats-report | 2 +- 34 files changed, 34 insertions(+), 34 deletions(-) diff --git a/bin/st b/bin/st index 4e6024f84f..b4b2aeeff6 100755 --- a/bin/st +++ b/bin/st @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/env python -u # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swauth-add-account b/bin/swauth-add-account index 9f40429d00..2b91b6292d 100755 --- a/bin/swauth-add-account +++ b/bin/swauth-add-account @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swauth-add-user b/bin/swauth-add-user index 365a24a085..23144df41b 100755 --- a/bin/swauth-add-user +++ b/bin/swauth-add-user @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swauth-cleanup-tokens b/bin/swauth-cleanup-tokens index 5666f801c2..3ca86cd990 100755 --- a/bin/swauth-cleanup-tokens +++ b/bin/swauth-cleanup-tokens @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swauth-delete-account b/bin/swauth-delete-account index 8e4bd42f76..66bdf2bbe1 100755 --- a/bin/swauth-delete-account +++ b/bin/swauth-delete-account @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swauth-delete-user b/bin/swauth-delete-user index f2f7808b64..de3ac3b12b 100755 --- a/bin/swauth-delete-user +++ b/bin/swauth-delete-user @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swauth-list b/bin/swauth-list index 217b5baff8..3f9ae5ea49 100755 --- a/bin/swauth-list +++ b/bin/swauth-list @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swauth-prep b/bin/swauth-prep index ca47ff9faa..a7b912e60c 100755 --- a/bin/swauth-prep +++ b/bin/swauth-prep @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swauth-set-account-service b/bin/swauth-set-account-service index 538210e497..0317546df5 100755 --- a/bin/swauth-set-account-service +++ b/bin/swauth-set-account-service @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-account-audit b/bin/swift-account-audit index 9a19198853..73441c1ba9 100755 --- a/bin/swift-account-audit +++ b/bin/swift-account-audit @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-account-auditor b/bin/swift-account-auditor index 3dc4c17609..f654dbce94 100755 --- a/bin/swift-account-auditor +++ b/bin/swift-account-auditor @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-account-reaper b/bin/swift-account-reaper index c5df6f4c2f..e318b386c7 100755 --- a/bin/swift-account-reaper +++ b/bin/swift-account-reaper @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-account-replicator b/bin/swift-account-replicator index 3978dc0bee..2adbd9ac46 100755 --- a/bin/swift-account-replicator +++ b/bin/swift-account-replicator @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-account-server b/bin/swift-account-server index a4088fffb2..6a1e459b76 100755 --- a/bin/swift-account-server +++ b/bin/swift-account-server @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-account-stats-logger b/bin/swift-account-stats-logger index 6256b690b5..450a5deca4 100755 --- a/bin/swift-account-stats-logger +++ b/bin/swift-account-stats-logger @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-bench b/bin/swift-bench index 0554782a06..7d20ef353a 100755 --- a/bin/swift-bench +++ b/bin/swift-bench @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-container-auditor b/bin/swift-container-auditor index 9c29d4400b..d445840c1d 100755 --- a/bin/swift-container-auditor +++ b/bin/swift-container-auditor @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-container-replicator b/bin/swift-container-replicator index d8443afacd..d88f069655 100755 --- a/bin/swift-container-replicator +++ b/bin/swift-container-replicator @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-container-server b/bin/swift-container-server index 2dbfbba090..e154c51633 100755 --- a/bin/swift-container-server +++ b/bin/swift-container-server @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-container-updater b/bin/swift-container-updater index d1b1d5ffb5..16ff3f4e62 100755 --- a/bin/swift-container-updater +++ b/bin/swift-container-updater @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-drive-audit b/bin/swift-drive-audit index 77912e720e..d269b25414 100755 --- a/bin/swift-drive-audit +++ b/bin/swift-drive-audit @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-get-nodes b/bin/swift-get-nodes index b84119222c..e7d13817af 100755 --- a/bin/swift-get-nodes +++ b/bin/swift-get-nodes @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-init b/bin/swift-init index 96ed1f63f3..7389a864a2 100644 --- a/bin/swift-init +++ b/bin/swift-init @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-log-stats-collector b/bin/swift-log-stats-collector index 374af49a4f..b1ec382570 100755 --- a/bin/swift-log-stats-collector +++ b/bin/swift-log-stats-collector @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-log-uploader b/bin/swift-log-uploader index 6db2bb01f9..11562699ee 100755 --- a/bin/swift-log-uploader +++ b/bin/swift-log-uploader @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-object-auditor b/bin/swift-object-auditor index c01b05bf70..53494f4cba 100755 --- a/bin/swift-object-auditor +++ b/bin/swift-object-auditor @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-object-info b/bin/swift-object-info index 278a7de0f2..f1b559be87 100755 --- a/bin/swift-object-info +++ b/bin/swift-object-info @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-object-replicator b/bin/swift-object-replicator index 7ae5db81c1..d84d892b4b 100755 --- a/bin/swift-object-replicator +++ b/bin/swift-object-replicator @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-object-server b/bin/swift-object-server index 3f36882c5f..225c21dffc 100755 --- a/bin/swift-object-server +++ b/bin/swift-object-server @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-object-updater b/bin/swift-object-updater index 779a3b7306..56048de081 100755 --- a/bin/swift-object-updater +++ b/bin/swift-object-updater @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-proxy-server b/bin/swift-proxy-server index 6dae54e156..66c8dee8ee 100755 --- a/bin/swift-proxy-server +++ b/bin/swift-proxy-server @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-ring-builder b/bin/swift-ring-builder index fd24a1d93f..f2618f70ef 100755 --- a/bin/swift-ring-builder +++ b/bin/swift-ring-builder @@ -1,4 +1,4 @@ -#!/usr/bin/python -uO +#!/usr/bin/env python -uO # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-stats-populate b/bin/swift-stats-populate index b1f4f0a568..d47117db26 100755 --- a/bin/swift-stats-populate +++ b/bin/swift-stats-populate @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/env python -u # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/swift-stats-report b/bin/swift-stats-report index 4c47b404de..3e4c21eabd 100755 --- a/bin/swift-stats-report +++ b/bin/swift-stats-report @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/env python -u # Copyright (c) 2010-2011 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); From 0e3040f9891f8880189fbed309e712f1348ef705 Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 24 Mar 2011 22:53:08 +0000 Subject: [PATCH 17/27] Changing some semantics and defaults; .r:* listings are off by default now and .rlistings turn them on; new x-container-meta-web-listings header to turn webmode listings on/off (off by default) --- swift/common/middleware/acl.py | 8 +- swift/common/middleware/staticweb.py | 105 +++++++++--------- swift/common/middleware/swauth.py | 2 +- test/unit/common/middleware/test_staticweb.py | 26 +++-- test/unit/common/middleware/test_swauth.py | 16 +-- 5 files changed, 84 insertions(+), 73 deletions(-) diff --git a/swift/common/middleware/acl.py b/swift/common/middleware/acl.py index e31ebda743..60d8bcffe2 100644 --- a/swift/common/middleware/acl.py +++ b/swift/common/middleware/acl.py @@ -58,9 +58,9 @@ def clean_acl(name, value): .r: .r:- - By default, allowing read access via .r will allow listing objects in the - container as well as retrieving objects from the container. To turn off - listings, use the .rnolisting directive. + By default, allowing read access via .r will not allow listing objects in + the container -- just retrieving objects from the container. To turn on + listings, use the .rlistings directive. Also, .r designations aren't allowed in headers whose names include the word 'write'. @@ -75,7 +75,7 @@ def clean_acl(name, value): ``bob,,,sue`` ``bob,sue`` ``.referrer : *`` ``.r:*`` ``.ref:*.example.com`` ``.r:.example.com`` - ``.r:*, .rnolisting`` ``.r:*,.rnolisting`` + ``.r:*, .rlistings`` ``.r:*,.rlistings`` ====================== ====================== :param name: The name of the header being cleaned, such as X-Container-Read diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index 248fa4fe72..d519c3db4a 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -34,20 +34,19 @@ added. For example:: [filter:staticweb] use = egg:swift#staticweb - # Seconds to cache container x-container-meta-index, - # x-container-meta-error, and x-container-listing-css header values. + # Seconds to cache container x-container-meta-web-* header values. # cache_timeout = 300 Any publicly readable containers (for example, ``X-Container-Read: .r:*``, see `acls`_ for more information on this) will be checked for -X-Container-Meta-Index and X-Container-Meta-Error header values:: +X-Container-Meta-Web-Index and X-Container-Meta-Web-Error header values:: - X-Container-Meta-Index - X-Container-Meta-Error + X-Container-Meta-Web-Index + X-Container-Meta-Web-Error -If X-Container-Meta-Index is set, any files will be served without -having to specify the part. For instance, setting -``X-Container-Meta-Index: index.html`` will be able to serve the object +If X-Container-Meta-Web-Index is set, any files will be served +without having to specify the part. For instance, setting +``X-Container-Meta-Web-Index: index.html`` will be able to serve the object .../pseudo/path/index.html with just .../pseudo/path or .../pseudo/path/ If X-Container-Meta-Error is set, any errors (currently just 401 Unauthorized @@ -55,17 +54,16 @@ and 404 Not Found) will instead serve the .../ object. For instance, setting ``X-Container-Meta-Error: error.html`` will serve .../404error.html for requests for paths not found. -For psuedo paths that have no , this middleware will serve HTML -file listings by default. If you don't want to serve such listings, you can -turn this off via the `acls`_ X-Container-Read setting of ``.rnolisting``. For -example, instead of ``X-Container-Read: .r:*`` you would use -``X-Container-Read: .r:*,.rnolisting`` +For psuedo paths that have no , this middleware can serve HTML file +listings if you set the ``X-Container-Meta-Web-Listings: true`` metadata item +on the container. If listings are enabled, the listings can have a custom style sheet by setting -the X-Container-Meta-Listing-CSS header. For instance, setting -``X-Container-Meta-Listing-CSS: listing.css`` will make listings link to the -.../listing.css style sheet. If you "view source" in your browser on a listing -page, you will see the well defined document structure that can be styled. +the X-Container-Meta-Web-Listings-CSS header. For instance, setting +``X-Container-Meta-Web-Listings-CSS: listing.css`` will make listings link to +the .../listing.css style sheet. If you "view source" in your browser on a +listing page, you will see the well defined document structure that can be +styled. Example usage of this middleware via ``st``: @@ -73,33 +71,32 @@ Example usage of this middleware via ``st``: st post -r '.r:*' container - You should be able to get objects and do direct container listings now, - though they'll be in the REST API format. + You should be able to get objects directly, but no index.html resolution or + listings. Set an index file directive:: - st post -m 'index:index.html' container + st post -m 'web-index:index.html' container You should be able to hit paths that have an index.html without needing to - type the index.html part and listings will now be HTML. + type the index.html part. - Turn off listings:: + Turn on listings:: - st post -r '.r:*,.rnolisting' container + st post -m 'web-listings: true' container + + Now you should see object listings for paths and pseudo paths that have no + index.html. + + Enable a custom listings style sheet:: + + st post -m 'web-listings-css:listings.css' container Set an error file:: - st post -m 'error:error.html' container + st post -m 'web-error:error.html' container Now 401's should load 401error.html, 404's should load 404error.html, etc. - - Turn listings back on:: - - st post -r '.r:*' container - - Enable a custom listing style sheet:: - - st post -m 'listing-css:listing.css' container """ @@ -132,16 +129,14 @@ class StaticWeb(object): self.app = app #: The filter configuration dict. self.conf = conf - #: The seconds to cache the x-container-meta-index, - #: x-container-meta-error, and x-container-listing-css headers for a - #: container. + #: The seconds to cache the x-container-meta-web-* headers., self.cache_timeout = int(conf.get('cache_timeout', 300)) # Results from the last call to self._start_response. self._response_status = None self._response_headers = None self._response_exc_info = None # Results from the last call to self._get_container_info. - self._index = self._error = self._listing_css = None + self._index = self._error = self._listings = self._listings_css = None def _start_response(self, status, headers, exc_info=None): """ @@ -196,22 +191,23 @@ class StaticWeb(object): def _get_container_info(self, env, start_response): """ - Retrieves x-container-meta-index, x-container-meta-error, and - x-container-meta-listing-css from memcache or from the cluster and - stores the result in memcache and in self._index, self._error, and - self._listing_css. + Retrieves x-container-meta-web-index, x-container-meta-web-error, + x-container-meta-web-listings, and x-container-meta-web-listings-css + from memcache or from the cluster and stores the result in memcache and + in self._index, self._error, self._listings, and self._listings_css. :param env: The WSGI environment dict. :param start_response: The WSGI start_response hook. """ - self._index = self._error = self._listing_css = None + self._index = self._error = self._listings = self._listings_css = None memcache_client = cache_from_env(env) if memcache_client: memcache_key = '/staticweb/%s/%s/%s' % (self.version, self.account, self.container) cached_data = memcache_client.get(memcache_key) if cached_data: - self._index, self._error, self._listing_css = cached_data + (self._index, self._error, self._listings, + self._listings_css) = cached_data return tmp_env = {'REQUEST_METHOD': 'HEAD', 'HTTP_USER_AGENT': 'StaticWeb'} for name in ('swift.cache', 'HTTP_X_CF_TRANS_ID'): @@ -222,14 +218,18 @@ class StaticWeb(object): resp = req.get_response(self.app) if resp.status_int // 100 == 2: self._index = \ - resp.headers.get('x-container-meta-index', '').strip() - self._listing_css = \ - resp.headers.get('x-container-meta-listing-css', '').strip() + resp.headers.get('x-container-meta-web-index', '').strip() self._error = \ - resp.headers.get('x-container-meta-error', '').strip() + resp.headers.get('x-container-meta-web-error', '').strip() + self._listings = \ + resp.headers.get('x-container-meta-web-listings', '').strip() + self._listings_css = \ + resp.headers.get('x-container-meta-web-listings-css', + '').strip() if memcache_client: memcache_client.set(memcache_key, - (self._index, self._error, self._listing_css), + (self._index, self._error, self._listings, + self._listings_css), timeout=self.cache_timeout) def _listing(self, env, start_response, prefix=None): @@ -240,6 +240,9 @@ class StaticWeb(object): :param start_response: The original WSGI start_response hook. :param prefix: Any prefix desired for the container listing. """ + if self._listings not in TRUE_VALUES: + resp = HTTPNotFound()(env, self._start_response) + return self._error_response(resp, env, start_response) tmp_env = dict(env) self._strip_ifs(tmp_env) tmp_env['REQUEST_METHOD'] = 'GET' @@ -262,11 +265,11 @@ class StaticWeb(object): ' \n' \ ' Listing of %s\n' % \ cgi.escape(env['PATH_INFO']) - if self._listing_css: + if self._listings_css: body += ' \n' % \ (self.version, self.account, self.container, - urllib.quote(self._listing_css)) + urllib.quote(self._listings_css)) else: body += '