From 7dde9096211eb5a0c116bdd3e9371734e9a39ab5 Mon Sep 17 00:00:00 2001 From: gholt Date: Sat, 5 May 2012 00:08:37 +0000 Subject: [PATCH] Pulled StaticWeb out to separate project StaticWeb is now at http://gholt.github.com/swift-staticweb/ For current users of StaticWeb, this will require installing the new package and changing the "use" line of the staticweb filter conf section to: use = egg:swiftstaticweb#middleware And then 'swift-init proxy reload'. Change-Id: Iab32adb5927698a667c5c6d6a572c44ca23414eb --- doc/manpages/proxy-server.conf.5 | 30 - doc/source/associated_projects.rst | 1 + doc/source/misc.rst | 7 - etc/proxy-server.conf-sample | 14 - setup.py | 1 - swift/common/middleware/staticweb.py | 564 ---------------- test/unit/common/middleware/test_staticweb.py | 628 ------------------ 7 files changed, 1 insertion(+), 1244 deletions(-) delete mode 100644 swift/common/middleware/staticweb.py delete mode 100644 test/unit/common/middleware/test_staticweb.py diff --git a/doc/manpages/proxy-server.conf.5 b/doc/manpages/proxy-server.conf.5 index 7a5273c3fe..109f053941 100644 --- a/doc/manpages/proxy-server.conf.5 +++ b/doc/manpages/proxy-server.conf.5 @@ -336,36 +336,6 @@ The default is 1. -.RS 0 -.IP "\fB[filter:staticweb]\fR" -.RE - -Note: Put staticweb just after your auth filter(s) in the pipeline - -.RS 3 -.IP \fBuse\fR -Entry point for paste.deploy for the staticweb middleware. This is the reference to the installed python egg. -The default is \fBegg:swift#staticweb\fR. -.IP \fBcache_timeout\fR -Seconds to cache container x-container-meta-web-* header values. The default is 300 seconds. -.IP "\fBset log_name\fR" -Label used when logging. The default is staticweb. -.IP "\fBset log_facility\fR" -Syslog log facility. The default is LOG_LOCAL0. -.IP "\fBset log_level\fR " -Logging level. The default is INFO. -.IP "\fBset log_headers\fR" -Enables the ability to log request headers. The default is False. -.IP "\fBset access_log_name\fR" -Label used when logging. The default is staticweb. -.IP "\fBset access_log_facility\fR" -Syslog log facility. The default is LOG_LOCAL0. -.IP "\fBset access_log_level\fR " -Logging level. The default is INFO. -.RE - - - .RS 0 .IP "\fB[filter:tempurl]\fR" .RE diff --git a/doc/source/associated_projects.rst b/doc/source/associated_projects.rst index 8b75f39368..4989296673 100644 --- a/doc/source/associated_projects.rst +++ b/doc/source/associated_projects.rst @@ -53,3 +53,4 @@ Other ----- * `Glance `_ - Provides services for discovering, registering, and retrieving virtual machine images (for OpenStack Compute [Nova], for example). +* `StaticWeb `_ - Allows serving static websites from Swift containers using ACLs and other metadata on those containers. diff --git a/doc/source/misc.rst b/doc/source/misc.rst index 8a3d3f2405..c462bd87e3 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -137,13 +137,6 @@ Swift3 :members: :show-inheritance: -StaticWeb -========= - -.. automodule:: swift.common.middleware.staticweb - :members: - :show-inheritance: - TempURL ======= diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index ee671603d7..6d3fed05d5 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -192,20 +192,6 @@ use = egg:swift#cname_lookup # storage_domain = example.com # lookup_depth = 1 -# Note: Put staticweb just after your auth filter(s) in the pipeline -[filter:staticweb] -use = egg:swift#staticweb -# Seconds to cache container x-container-meta-web-* header values. -# cache_timeout = 300 -# You can override the default log routing for this filter here: -# set log_name = staticweb -# set log_facility = LOG_LOCAL0 -# set log_level = INFO -# set access_log_name = staticweb -# set access_log_facility = LOG_LOCAL0 -# set access_log_level = INFO -# set log_headers = False - # Note: Put tempurl just before your auth filter(s) in the pipeline [filter:tempurl] use = egg:swift#tempurl diff --git a/setup.py b/setup.py index d195d34f6a..66bf89d8db 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,6 @@ setup( 'catch_errors=swift.common.middleware.catch_errors:filter_factory', 'domain_remap=swift.common.middleware.domain_remap:filter_factory', 'swift3=swift.common.middleware.swift3:filter_factory', - 'staticweb=swift.common.middleware.staticweb:filter_factory', 'tempauth=swift.common.middleware.tempauth:filter_factory', 'recon=swift.common.middleware.recon:filter_factory', 'tempurl=swift.common.middleware.tempurl:filter_factory', diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py deleted file mode 100644 index 547e82368a..0000000000 --- a/swift/common/middleware/staticweb.py +++ /dev/null @@ -1,564 +0,0 @@ -# Copyright (c) 2010-2012 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This StaticWeb WSGI middleware will serve container data as a static web site -with index file and error file resolution and optional file listings. This mode -is normally only active for anonymous requests. If you want to use it with -authenticated requests, set the ``X-Web-Mode: true`` header on the request. - -The ``staticweb`` filter should be added to the pipeline in your -``/etc/swift/proxy-server.conf`` file just after any auth middleware. Also, the -configuration section for the ``staticweb`` middleware itself needs to be -added. For example:: - - [DEFAULT] - ... - - [pipeline:main] - pipeline = healthcheck cache tempauth staticweb proxy-server - - ... - - [filter:staticweb] - use = egg:swift#staticweb - # Seconds to cache container x-container-meta-web-* header values. - # cache_timeout = 300 - # You can override the default log routing for this filter here: - # set log_name = staticweb - # set log_facility = LOG_LOCAL0 - # set log_level = INFO - # set access_log_name = staticweb - # set access_log_facility = LOG_LOCAL0 - # set access_log_level = INFO - # set log_headers = False - -Any publicly readable containers (for example, ``X-Container-Read: .r:*``, see -`acls`_ for more information on this) will be checked for -X-Container-Meta-Web-Index and X-Container-Meta-Web-Error header values:: - - X-Container-Meta-Web-Index - X-Container-Meta-Web-Error - -If X-Container-Meta-Web-Index is set, any files will be served -without having to specify the part. For instance, setting -``X-Container-Meta-Web-Index: index.html`` will be able to serve the object -.../pseudo/path/index.html with just .../pseudo/path or .../pseudo/path/ - -If X-Container-Meta-Web-Error is set, any errors (currently just 401 -Unauthorized and 404 Not Found) will instead serve the -.../ object. For instance, setting -``X-Container-Meta-Web-Error: error.html`` will serve .../404error.html for -requests for paths not found. - -For psuedo paths that have no , this middleware can serve HTML file -listings if you set the ``X-Container-Meta-Web-Listings: true`` metadata item -on the container. - -If listings are enabled, the listings can have a custom style sheet by setting -the X-Container-Meta-Web-Listings-CSS header. For instance, setting -``X-Container-Meta-Web-Listings-CSS: listing.css`` will make listings link to -the .../listing.css style sheet. If you "view source" in your browser on a -listing page, you will see the well defined document structure that can be -styled. - -Example usage of this middleware via ``swift``: - - Make the container publicly readable:: - - swift post -r '.r:*' container - - You should be able to get objects directly, but no index.html resolution or - listings. - - Set an index file directive:: - - swift post -m 'web-index:index.html' container - - You should be able to hit paths that have an index.html without needing to - type the index.html part. - - Turn on listings:: - - swift post -m 'web-listings: true' container - - Now you should see object listings for paths and pseudo paths that have no - index.html. - - Enable a custom listings style sheet:: - - swift post -m 'web-listings-css:listings.css' container - - Set an error file:: - - swift post -m 'web-error:error.html' container - - Now 401's should load 401error.html, 404's should load 404error.html, etc. -""" - - -try: - import simplejson as json -except ImportError: - import json - -import cgi -import time -from urllib import unquote, quote as urllib_quote - -from webob import Response -from webob.exc import HTTPMovedPermanently, HTTPNotFound - -from swift.common.utils import cache_from_env, get_logger, human_readable, \ - split_path, TRUE_VALUES -from swift.common.wsgi import make_pre_authed_env, make_pre_authed_request, \ - WSGIContext -from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND - - -def quote(value, safe='/'): - """ - Patched version of urllib.quote that encodes utf-8 strings before quoting - """ - if isinstance(value, unicode): - value = value.encode('utf-8') - return urllib_quote(value, safe) - - -class _StaticWebContext(WSGIContext): - """ - The Static Web WSGI middleware filter; serves container data as a - static web site. See `staticweb`_ for an overview. - - This _StaticWebContext is used by StaticWeb with each request - that might need to be handled to make keeping contextual - information about the request a bit simpler than storing it in - the WSGI env. - """ - - def __init__(self, staticweb, version, account, container, obj): - WSGIContext.__init__(self, staticweb.app) - self.version = version - self.account = account - self.container = container - self.obj = obj - self.app = staticweb.app - self.cache_timeout = staticweb.cache_timeout - self.logger = staticweb.logger - self.access_logger = staticweb.access_logger - self.log_headers = staticweb.log_headers - self.agent = '%(orig)s StaticWeb' - # Results from the last call to self._get_container_info. - self._index = self._error = self._listings = self._listings_css = None - - def _error_response(self, response, env, start_response): - """ - Sends the error response to the remote client, possibly resolving a - custom error response body based on x-container-meta-web-error. - - :param response: The error response we should default to sending. - :param env: The original request WSGI environment. - :param start_response: The WSGI start_response hook. - """ - self._log_response(env, self._get_status_int()) - if not self._error: - start_response(self._response_status, self._response_headers, - self._response_exc_info) - return response - save_response_status = self._response_status - save_response_headers = self._response_headers - save_response_exc_info = self._response_exc_info - resp = self._app_call(make_pre_authed_env(env, 'GET', - '/%s/%s/%s/%s%s' % (self.version, self.account, self.container, - self._get_status_int(), self._error), - self.agent)) - if is_success(self._get_status_int()): - start_response(save_response_status, self._response_headers, - self._response_exc_info) - return resp - start_response(save_response_status, save_response_headers, - save_response_exc_info) - return response - - def _get_container_info(self, env): - """ - Retrieves x-container-meta-web-index, x-container-meta-web-error, - x-container-meta-web-listings, and x-container-meta-web-listings-css - from memcache or from the cluster and stores the result in memcache and - in self._index, self._error, self._listings, and self._listings_css. - - :param env: The WSGI environment dict. - """ - self._index = self._error = self._listings = self._listings_css = None - memcache_client = cache_from_env(env) - if memcache_client: - memcache_key = '/staticweb/%s/%s/%s' % (self.version, self.account, - self.container) - cached_data = memcache_client.get(memcache_key) - if cached_data: - (self._index, self._error, self._listings, - self._listings_css) = cached_data - return - resp = make_pre_authed_request(env, 'HEAD', - '/%s/%s/%s' % (self.version, self.account, self.container), - agent=self.agent).get_response(self.app) - if is_success(resp.status_int): - self._index = \ - resp.headers.get('x-container-meta-web-index', '').strip() - self._error = \ - resp.headers.get('x-container-meta-web-error', '').strip() - self._listings = \ - resp.headers.get('x-container-meta-web-listings', '').strip() - self._listings_css = \ - resp.headers.get('x-container-meta-web-listings-css', - '').strip() - if memcache_client: - memcache_client.set(memcache_key, - (self._index, self._error, self._listings, - self._listings_css), - timeout=self.cache_timeout) - - def _listing(self, env, start_response, prefix=None): - """ - Sends an HTML object listing to the remote client. - - :param env: The original WSGI environment dict. - :param start_response: The original WSGI start_response hook. - :param prefix: Any prefix desired for the container listing. - """ - if self._listings.lower() not in TRUE_VALUES: - resp = HTTPNotFound()(env, self._start_response) - return self._error_response(resp, env, start_response) - tmp_env = make_pre_authed_env(env, 'GET', - '/%s/%s/%s' % (self.version, self.account, self.container), - self.agent) - tmp_env['QUERY_STRING'] = 'delimiter=/&format=json' - if prefix: - tmp_env['QUERY_STRING'] += '&prefix=%s' % quote(prefix) - else: - prefix = '' - resp = self._app_call(tmp_env) - if not is_success(self._get_status_int()): - return self._error_response(resp, env, start_response) - listing = None - body = ''.join(resp) - if body: - listing = json.loads(body) - if not listing: - resp = HTTPNotFound()(env, self._start_response) - return self._error_response(resp, env, start_response) - headers = {'Content-Type': 'text/html; charset=UTF-8'} - body = '\n' \ - '\n' \ - ' \n' \ - ' Listing of %s\n' % \ - cgi.escape(env['PATH_INFO']) - if self._listings_css: - body += ' \n' % (self._build_css_path(prefix)) - else: - body += ' \n' - body += ' \n' \ - ' \n' \ - '

Listing of %s

\n' \ - ' \n' \ - ' \n' \ - ' \n' \ - ' \n' \ - ' \n' \ - ' \n' % \ - cgi.escape(env['PATH_INFO']) - if prefix: - body += ' \n' \ - ' \n' \ - ' \n' \ - ' \n' \ - ' \n' - for item in listing: - if 'subdir' in item: - subdir = item['subdir'] - if prefix: - subdir = subdir[len(prefix):] - body += ' \n' \ - ' \n' \ - ' \n' \ - ' \n' \ - ' \n' % \ - (quote(subdir), cgi.escape(subdir)) - for item in listing: - if 'name' in item: - name = item['name'] - if prefix: - name = name[len(prefix):] - body += ' \n' \ - ' \n' \ - ' \n' \ - ' \n' \ - ' \n' % \ - (' '.join('type-' + cgi.escape(t.lower(), quote=True) - for t in item['content_type'].split('/')), - quote(name), cgi.escape(name), - human_readable(item['bytes']), - cgi.escape(item['last_modified']).split('.')[0]. - replace('T', ' ')) - body += '
NameSizeDate
../  
%s  
%s%s%s
\n' \ - ' \n' \ - '\n' - resp = Response(headers=headers, body=body) - self._log_response(env, resp.status_int) - return resp(env, start_response) - - def _build_css_path(self, prefix=''): - """ - Constructs a relative path from a given prefix within the container. - URLs and paths starting with '/' are not modified. - - :param prefix: The prefix for the container listing. - """ - if self._listings_css.startswith(('/', 'http://', 'https://')): - css_path = quote(self._listings_css, ':/') - else: - css_path = '../' * prefix.count('/') + quote(self._listings_css) - return css_path - - def handle_container(self, env, start_response): - """ - Handles a possible static web request for a container. - - :param env: The original WSGI environment dict. - :param start_response: The original WSGI start_response hook. - """ - self._get_container_info(env) - if not self._listings and not self._index: - if env.get('HTTP_X_WEB_MODE', 'f').lower() in TRUE_VALUES: - return HTTPNotFound()(env, start_response) - return self.app(env, start_response) - if env['PATH_INFO'][-1] != '/': - resp = HTTPMovedPermanently( - location=(env['PATH_INFO'] + '/')) - self._log_response(env, resp.status_int) - return resp(env, start_response) - if not self._index: - return self._listing(env, start_response) - tmp_env = dict(env) - tmp_env['HTTP_USER_AGENT'] = \ - '%s StaticWeb' % env.get('HTTP_USER_AGENT') - tmp_env['PATH_INFO'] += self._index - resp = self._app_call(tmp_env) - status_int = self._get_status_int() - if status_int == HTTP_NOT_FOUND: - return self._listing(env, start_response) - elif not is_success(self._get_status_int()) or \ - not is_redirection(self._get_status_int()): - return self._error_response(resp, env, start_response) - start_response(self._response_status, self._response_headers, - self._response_exc_info) - return resp - - def handle_object(self, env, start_response): - """ - Handles a possible static web request for an object. This object could - resolve into an index or listing request. - - :param env: The original WSGI environment dict. - :param start_response: The original WSGI start_response hook. - """ - tmp_env = dict(env) - tmp_env['HTTP_USER_AGENT'] = \ - '%s StaticWeb' % env.get('HTTP_USER_AGENT') - resp = self._app_call(tmp_env) - status_int = self._get_status_int() - if is_success(status_int) or is_redirection(status_int): - start_response(self._response_status, self._response_headers, - self._response_exc_info) - return resp - if status_int != HTTP_NOT_FOUND: - return self._error_response(resp, env, start_response) - self._get_container_info(env) - if not self._listings and not self._index: - return self.app(env, start_response) - status_int = HTTP_NOT_FOUND - if self._index: - tmp_env = dict(env) - tmp_env['HTTP_USER_AGENT'] = \ - '%s StaticWeb' % env.get('HTTP_USER_AGENT') - if tmp_env['PATH_INFO'][-1] != '/': - tmp_env['PATH_INFO'] += '/' - tmp_env['PATH_INFO'] += self._index - resp = self._app_call(tmp_env) - status_int = self._get_status_int() - if is_success(status_int) or is_redirection(status_int): - if env['PATH_INFO'][-1] != '/': - resp = HTTPMovedPermanently( - location=env['PATH_INFO'] + '/') - self._log_response(env, resp.status_int) - return resp(env, start_response) - start_response(self._response_status, self._response_headers, - self._response_exc_info) - return resp - if status_int == HTTP_NOT_FOUND: - if env['PATH_INFO'][-1] != '/': - tmp_env = make_pre_authed_env(env, 'GET', - '/%s/%s/%s' % (self.version, self.account, - self.container), - self.agent) - tmp_env['QUERY_STRING'] = 'limit=1&format=json&delimiter' \ - '=/&limit=1&prefix=%s' % quote(self.obj + '/') - resp = self._app_call(tmp_env) - body = ''.join(resp) - if not is_success(self._get_status_int()) or not body or \ - not json.loads(body): - resp = HTTPNotFound()(env, self._start_response) - return self._error_response(resp, env, start_response) - resp = HTTPMovedPermanently(location=env['PATH_INFO'] + - '/') - self._log_response(env, resp.status_int) - return resp(env, start_response) - return self._listing(env, start_response, self.obj) - - def _log_response(self, env, status_int): - """ - Logs an access line for StaticWeb responses; use when the next app in - the pipeline will not be handling the final response to the remote - user. - - Assumes that the request and response bodies are 0 bytes or very near 0 - so no bytes transferred are tracked or logged. - - This does mean that the listings responses that actually do transfer - content will not be logged with any bytes transferred, but in counter - to that the full bytes for the underlying listing will be logged by the - proxy even if the remote client disconnects early for the StaticWeb - listing. - - I didn't think the extra complexity of getting the bytes transferred - exactly correct for these requests was worth it, but perhaps someone - else will think it is. - - To get things exact, this filter would need to use an - eventlet.posthooks logger like the proxy does and any log processing - systems would need to ignore some (but not all) proxy requests made by - StaticWeb if they were just interested in the bytes transferred to the - remote client. - """ - trans_time = '%.4f' % (time.time() - - env.get('staticweb.start_time', time.time())) - the_request = quote(unquote(env['PATH_INFO'])) - if env.get('QUERY_STRING'): - the_request = the_request + '?' + env['QUERY_STRING'] - # remote user for zeus - client = env.get('HTTP_X_CLUSTER_CLIENT_IP') - if not client and 'HTTP_X_FORWARDED_FOR' in env: - # remote user for other lbs - client = env['HTTP_X_FORWARDED_FOR'].split(',')[0].strip() - logged_headers = None - if self.log_headers: - logged_headers = '\n'.join('%s: %s' % (k, v) - for k, v in req.headers.items()) - self.access_logger.info(' '.join(quote(str(x)) for x in ( - client or '-', - env.get('REMOTE_ADDR', '-'), - time.strftime('%d/%b/%Y/%H/%M/%S', time.gmtime()), - env['REQUEST_METHOD'], - the_request, - env['SERVER_PROTOCOL'], - status_int, - env.get('HTTP_REFERER', '-'), - env.get('HTTP_USER_AGENT', '-'), - env.get('HTTP_X_AUTH_TOKEN', '-'), - '-', - '-', - env.get('HTTP_ETAG', '-'), - env.get('swift.trans_id', '-'), - logged_headers or '-', - trans_time))) - - -class StaticWeb(object): - """ - The Static Web WSGI middleware filter; serves container data as a static - web site. See `staticweb`_ for an overview. - - :param app: The next WSGI application/filter in the paste.deploy pipeline. - :param conf: The filter configuration dict. - """ - - def __init__(self, app, conf): - #: The next WSGI application/filter in the paste.deploy pipeline. - self.app = app - #: The filter configuration dict. - self.conf = conf - #: The seconds to cache the x-container-meta-web-* headers., - self.cache_timeout = int(conf.get('cache_timeout', 300)) - #: Logger for this filter. - self.logger = get_logger(conf, log_route='staticweb') - access_log_conf = {} - for key in ('log_facility', 'log_name', 'log_level'): - value = conf.get('access_' + key, conf.get(key, None)) - if value: - access_log_conf[key] = value - #: Web access logger for this filter. - self.access_logger = get_logger(access_log_conf, - log_route='staticweb-access') - #: Indicates whether full HTTP headers should be logged or not. - self.log_headers = conf.get('log_headers', 'f').lower() in TRUE_VALUES - - def __call__(self, env, start_response): - """ - Main hook into the WSGI paste.deploy filter/app pipeline. - - :param env: The WSGI environment dict. - :param start_response: The WSGI start_response hook. - """ - env['staticweb.start_time'] = time.time() - try: - (version, account, container, obj) = \ - split_path(env['PATH_INFO'], 2, 4, True) - except ValueError: - return self.app(env, start_response) - if env['REQUEST_METHOD'] in ('PUT', 'POST') and container and not obj: - memcache_client = cache_from_env(env) - if memcache_client: - memcache_key = \ - '/staticweb/%s/%s/%s' % (version, account, container) - memcache_client.delete(memcache_key) - return self.app(env, start_response) - if env['REQUEST_METHOD'] not in ('HEAD', 'GET'): - return self.app(env, start_response) - if env.get('REMOTE_USER') and \ - env.get('HTTP_X_WEB_MODE', 'f').lower() not in TRUE_VALUES: - return self.app(env, start_response) - if not container: - return self.app(env, start_response) - context = _StaticWebContext(self, version, account, container, obj) - if obj: - return context.handle_object(env, start_response) - return context.handle_container(env, start_response) - - -def filter_factory(global_conf, **local_conf): - """ Returns a Static Web WSGI filter for use with paste.deploy. """ - conf = global_conf.copy() - conf.update(local_conf) - - def staticweb_filter(app): - return StaticWeb(app, conf) - return staticweb_filter diff --git a/test/unit/common/middleware/test_staticweb.py b/test/unit/common/middleware/test_staticweb.py deleted file mode 100644 index 8602e8d664..0000000000 --- a/test/unit/common/middleware/test_staticweb.py +++ /dev/null @@ -1,628 +0,0 @@ -# Copyright (c) 2010 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -try: - import simplejson as json -except ImportError: - import json -import unittest -from contextlib import contextmanager - -from webob import Request, Response - -from swift.common.middleware import staticweb - - -class FakeMemcache(object): - - def __init__(self): - self.store = {} - - def get(self, key): - return self.store.get(key) - - def set(self, key, value, timeout=0): - self.store[key] = value - return True - - def incr(self, key, timeout=0): - self.store[key] = self.store.setdefault(key, 0) + 1 - return self.store[key] - - @contextmanager - def soft_lock(self, key, timeout=0, retries=5): - yield True - - def delete(self, key): - try: - del self.store[key] - except Exception: - pass - return True - - -class FakeApp(object): - - def __init__(self, status_headers_body_iter=None): - self.calls = 0 - self.get_c4_called = False - - def __call__(self, env, start_response): - self.calls += 1 - if env['PATH_INFO'] == '/': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1': - return Response( - status='412 Precondition Failed')(env, start_response) - elif env['PATH_INFO'] == '/v1/a': - return Response(status='401 Unauthorized')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c1': - return Response(status='401 Unauthorized')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c2': - return self.listing(env, start_response, - {'x-container-read': '.r:*'}) - elif env['PATH_INFO'] == '/v1/a/c2/one.txt': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3': - return self.listing(env, start_response, - {'x-container-read': '.r:*', - 'x-container-meta-web-index': 'index.html', - 'x-container-meta-web-listings': 't'}) - elif env['PATH_INFO'] == '/v1/a/c3/index.html': - return Response(status='200 Ok', body=''' - - -

Test main index.html file.

-

Visit subdir.

-

Don't visit subdir2 because it doesn't really - exist.

-

Visit subdir3.

-

Visit subdir3/subsubdir.

- - - ''')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3b': - return self.listing(env, start_response, - {'x-container-read': '.r:*', - 'x-container-meta-web-index': 'index.html', - 'x-container-meta-web-listings': 't'}) - elif env['PATH_INFO'] == '/v1/a/c3b/index.html': - resp = Response(status='204 No Content') - resp.app_iter = iter([]) - return resp(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3/subdir': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3/subdir/': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3/subdir/index.html': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3/subdir3/subsubdir': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3/subdir3/subsubdir/': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3/subdir3/subsubdir/index.html': - return Response(status='200 Ok', body='index file')(env, - start_response) - elif env['PATH_INFO'] == '/v1/a/c3/subdirx/': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3/subdirx/index.html': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3/subdiry/': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3/subdiry/index.html': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3/subdirz': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3/subdirz/index.html': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3/unknown': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c3/unknown/index.html': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c4': - self.get_c4_called = True - return self.listing(env, start_response, - {'x-container-read': '.r:*', - 'x-container-meta-web-index': 'index.html', - 'x-container-meta-web-error': 'error.html', - 'x-container-meta-web-listings': 't', - 'x-container-meta-web-listings-css': 'listing.css'}) - elif env['PATH_INFO'] == '/v1/a/c4/one.txt': - return Response(status='200 Ok', - headers={'x-object-meta-test': 'value'}, - body='1')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c4/two.txt': - return Response(status='503 Service Unavailable')(env, - start_response) - elif env['PATH_INFO'] == '/v1/a/c4/index.html': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c4/subdir/': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c4/subdir/index.html': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c4/unknown': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c4/unknown/index.html': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c4/404error.html': - return Response(status='200 Ok', body=''' - - -

Chrome's 404 fancy-page sucks.

- - - '''.strip())(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c5': - return self.listing(env, start_response, - {'x-container-read': '.r:*', - 'x-container-meta-web-index': 'index.html', - 'x-container-meta-listings': 't', - 'x-container-meta-web-error': 'error.html'}) - elif env['PATH_INFO'] == '/v1/a/c5/index.html': - return Response(status='503 Service Unavailable')(env, - start_response) - elif env['PATH_INFO'] == '/v1/a/c5/503error.html': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c5/unknown': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c5/unknown/index.html': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c5/404error.html': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] == '/v1/a/c6': - return self.listing(env, start_response, - {'x-container-read': '.r:*', - 'x-container-meta-web-listings': 't'}) - elif env['PATH_INFO'] == '/v1/a/c6/subdir': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] in ('/v1/a/c7', '/v1/a/c7/'): - return self.listing(env, start_response, - {'x-container-read': '.r:*', - 'x-container-meta-web-listings': 'f'}) - elif env['PATH_INFO'] in ('/v1/a/c8', '/v1/a/c8/'): - return self.listing(env, start_response, - {'x-container-read': '.r:*', - 'x-container-meta-web-error': 'error.html', - 'x-container-meta-web-listings': 't', - 'x-container-meta-web-listings-css': \ - 'http://localhost/stylesheets/listing.css'}) - elif env['PATH_INFO'] == '/v1/a/c8/subdir/': - return Response(status='404 Not Found')(env, start_response) - elif env['PATH_INFO'] in ('/v1/a/c9', '/v1/a/c9/'): - return self.listing(env, start_response, - {'x-container-read': '.r:*', - 'x-container-meta-web-error': 'error.html', - 'x-container-meta-web-listings': 't', - 'x-container-meta-web-listings-css': \ - '/absolute/listing.css'}) - elif env['PATH_INFO'] == '/v1/a/c9/subdir/': - return Response(status='404 Not Found')(env, start_response) - else: - raise Exception('Unknown path %r' % env['PATH_INFO']) - - def listing(self, env, start_response, headers): - if env['PATH_INFO'] in ('/v1/a/c3', '/v1/a/c4', '/v1/a/c8', \ - '/v1/a/c9') and \ - env['QUERY_STRING'] == 'delimiter=/&format=json&prefix=subdir/': - headers.update({'X-Container-Object-Count': '11', - 'X-Container-Bytes-Used': '73741', - 'X-Container-Read': '.r:*', - 'Content-Type': 'application/json; charset=utf-8'}) - body = ''' - [{"name":"subdir/1.txt", - "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, - "content_type":"text/plain", - "last_modified":"2011-03-24T04:27:52.709100"}, - {"name":"subdir/2.txt", - "hash":"c85c1dcd19cf5cbac84e6043c31bb63e", "bytes":20, - "content_type":"text/plain", - "last_modified":"2011-03-24T04:27:52.734140"}, - {"subdir":"subdir3/subsubdir/"}] - '''.strip() - elif env['PATH_INFO'] == '/v1/a/c3' and env['QUERY_STRING'] == \ - 'delimiter=/&format=json&prefix=subdiry/': - headers.update({'X-Container-Object-Count': '11', - 'X-Container-Bytes-Used': '73741', - 'X-Container-Read': '.r:*', - 'Content-Type': 'application/json; charset=utf-8'}) - body = '[]' - elif env['PATH_INFO'] == '/v1/a/c3' and env['QUERY_STRING'] == \ - 'limit=1&format=json&delimiter=/&limit=1&prefix=subdirz/': - headers.update({'X-Container-Object-Count': '11', - 'X-Container-Bytes-Used': '73741', - 'X-Container-Read': '.r:*', - 'Content-Type': 'application/json; charset=utf-8'}) - body = ''' - [{"name":"subdirz/1.txt", - "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, - "content_type":"text/plain", - "last_modified":"2011-03-24T04:27:52.709100"}] - '''.strip() - elif env['PATH_INFO'] == '/v1/a/c6' and env['QUERY_STRING'] == \ - 'limit=1&format=json&delimiter=/&limit=1&prefix=subdir/': - headers.update({'X-Container-Object-Count': '11', - 'X-Container-Bytes-Used': '73741', - 'X-Container-Read': '.r:*', - 'X-Container-Web-Listings': 't', - 'Content-Type': 'application/json; charset=utf-8'}) - body = ''' - [{"name":"subdir/1.txt", - "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, - "content_type":"text/plain", - "last_modified":"2011-03-24T04:27:52.709100"}] - '''.strip() - elif 'prefix=' in env['QUERY_STRING']: - return Response(status='204 No Content')(env, start_response) - elif 'format=json' in env['QUERY_STRING']: - headers.update({'X-Container-Object-Count': '11', - 'X-Container-Bytes-Used': '73741', - 'Content-Type': 'application/json; charset=utf-8'}) - body = ''' - [{"name":"401error.html", - "hash":"893f8d80692a4d3875b45be8f152ad18", "bytes":110, - "content_type":"text/html", - "last_modified":"2011-03-24T04:27:52.713710"}, - {"name":"404error.html", - "hash":"62dcec9c34ed2b347d94e6ca707aff8c", "bytes":130, - "content_type":"text/html", - "last_modified":"2011-03-24T04:27:52.720850"}, - {"name":"index.html", - "hash":"8b469f2ca117668a5131fe9ee0815421", "bytes":347, - "content_type":"text/html", - "last_modified":"2011-03-24T04:27:52.683590"}, - {"name":"listing.css", - "hash":"7eab5d169f3fcd06a08c130fa10c5236", "bytes":17, - "content_type":"text/css", - "last_modified":"2011-03-24T04:27:52.721610"}, - {"name":"one.txt", "hash":"73f1dd69bacbf0847cc9cffa3c6b23a1", - "bytes":22, "content_type":"text/plain", - "last_modified":"2011-03-24T04:27:52.722270"}, - {"name":"subdir/1.txt", - "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, - "content_type":"text/plain", - "last_modified":"2011-03-24T04:27:52.709100"}, - {"name":"subdir/2.txt", - "hash":"c85c1dcd19cf5cbac84e6043c31bb63e", "bytes":20, - "content_type":"text/plain", - "last_modified":"2011-03-24T04:27:52.734140"}, - {"name":"subdir/\u2603.txt", - "hash":"7337d028c093130898d937c319cc9865", "bytes":72981, - "content_type":"text/plain", - "last_modified":"2011-03-24T04:27:52.735460"}, - {"name":"subdir2", "hash":"d41d8cd98f00b204e9800998ecf8427e", - "bytes":0, "content_type":"text/directory", - "last_modified":"2011-03-24T04:27:52.676690"}, - {"name":"subdir3/subsubdir/index.html", - "hash":"04eea67110f883b1a5c97eb44ccad08c", "bytes":72, - "content_type":"text/html", - "last_modified":"2011-03-24T04:27:52.751260"}, - {"name":"two.txt", "hash":"10abb84c63a5cff379fdfd6385918833", - "bytes":22, "content_type":"text/plain", - "last_modified":"2011-03-24T04:27:52.825110"}] - '''.strip() - else: - headers.update({'X-Container-Object-Count': '11', - 'X-Container-Bytes-Used': '73741', - 'Content-Type': 'text/plain; charset=utf-8'}) - body = '\n'.join(['401error.html', '404error.html', 'index.html', - 'listing.css', 'one.txt', 'subdir/1.txt', - 'subdir/2.txt', u'subdir/\u2603.txt', 'subdir2', - 'subdir3/subsubdir/index.html', 'two.txt']) - return Response(status='200 Ok', headers=headers, - body=body)(env, start_response) - - -class TestStaticWeb(unittest.TestCase): - - def setUp(self): - self.app = FakeApp() - self.test_staticweb = staticweb.filter_factory({})(self.app) - - def test_app_set(self): - app = FakeApp() - sw = staticweb.filter_factory({})(app) - self.assertEquals(sw.app, app) - - def test_conf_set(self): - conf = {'blah': 1} - sw = staticweb.filter_factory(conf)(FakeApp()) - self.assertEquals(sw.conf, conf) - - def test_cache_timeout_unset(self): - sw = staticweb.filter_factory({})(FakeApp()) - self.assertEquals(sw.cache_timeout, 300) - - def test_cache_timeout_set(self): - sw = staticweb.filter_factory({'cache_timeout': '1'})(FakeApp()) - self.assertEquals(sw.cache_timeout, 1) - - def test_root(self): - resp = Request.blank('/').get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 404) - - def test_version(self): - resp = Request.blank('/v1').get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 412) - - def test_account(self): - resp = Request.blank('/v1/a').get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 401) - - def test_container1(self): - resp = Request.blank('/v1/a/c1').get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 401) - - def test_container1_web_mode_explicitly_off(self): - resp = Request.blank('/v1/a/c1', - headers={'x-web-mode': 'false'}).get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 401) - - def test_container1_web_mode_explicitly_on(self): - resp = Request.blank('/v1/a/c1', - headers={'x-web-mode': 'true'}).get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 404) - - def test_container2(self): - resp = Request.blank('/v1/a/c2').get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 200) - self.assertEquals(resp.content_type, 'text/plain') - self.assertEquals(len(resp.body.split('\n')), - int(resp.headers['x-container-object-count'])) - - def test_container2_web_mode_explicitly_off(self): - resp = Request.blank('/v1/a/c2', - headers={'x-web-mode': 'false'}).get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 200) - self.assertEquals(resp.content_type, 'text/plain') - self.assertEquals(len(resp.body.split('\n')), - int(resp.headers['x-container-object-count'])) - - def test_container2_web_mode_explicitly_on(self): - resp = Request.blank('/v1/a/c2', - headers={'x-web-mode': 'true'}).get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 404) - - def test_container2onetxt(self): - resp = Request.blank( - '/v1/a/c2/one.txt').get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 404) - - def test_container2json(self): - resp = Request.blank( - '/v1/a/c2?format=json').get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 200) - self.assertEquals(resp.content_type, 'application/json') - self.assertEquals(len(json.loads(resp.body)), - int(resp.headers['x-container-object-count'])) - - def test_container2json_web_mode_explicitly_off(self): - resp = Request.blank('/v1/a/c2?format=json', - headers={'x-web-mode': 'false'}).get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 200) - self.assertEquals(resp.content_type, 'application/json') - self.assertEquals(len(json.loads(resp.body)), - int(resp.headers['x-container-object-count'])) - - def test_container2json_web_mode_explicitly_on(self): - resp = Request.blank('/v1/a/c2?format=json', - headers={'x-web-mode': 'true'}).get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 404) - - def test_container3(self): - resp = Request.blank('/v1/a/c3').get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 301) - self.assertEquals(resp.headers['location'], - 'http://localhost/v1/a/c3/') - - def test_container3indexhtml(self): - resp = Request.blank('/v1/a/c3/').get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 200) - self.assert_('Test main index.html file.' in resp.body) - - def test_container3subdir(self): - resp = Request.blank( - '/v1/a/c3/subdir').get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 301) - - def test_container3subsubdir(self): - resp = Request.blank( - '/v1/a/c3/subdir3/subsubdir').get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 301) - - def test_container3subsubdircontents(self): - resp = Request.blank( - '/v1/a/c3/subdir3/subsubdir/').get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 200) - self.assertEquals(resp.body, 'index file') - - def test_container3subdir(self): - resp = Request.blank( - '/v1/a/c3/subdir/').get_response(self.test_staticweb) - self.assertEquals(resp.status_int, 200) - self.assert_('Listing of /v1/a/c3/subdir/' in resp.body) - self.assert_('' in resp.body) - self.assert_('' not in resp.body) - self.assert_('