From 6164fa246d4c8d2613b44751e3f08330694d1497 Mon Sep 17 00:00:00 2001 From: anc Date: Tue, 3 Dec 2013 22:02:39 +0000 Subject: [PATCH] Generic means for persisting system metadata. Middleware or core features may need to store metadata against accounts or containers. This patch adds a generic mechanism for system metadata to be persisted in backend databases, without polluting the user metadata namespace, by using the reserved header namespace x--sysmeta-*. Modifications are firstly that backend servers persist system metadata headers alongside user metadata and other system state. For accounts and containers, system metadata in PUT and POST requests is treated in a similar way to user metadata. System metadata is not yet supported for object requests. Secondly, changes in the proxy controllers ensure that headers in the system metadata namespace will pass through in requests to backend servers. Thirdly, system metadata returned from backend servers in GET or HEAD responses is added to the cached info dict, which middleware can access. Finally, a gatekeeper middleware module is provided which filters all system metadata headers from requests and responses by removing headers with names starting x-account-sysmeta-, x-container-sysmeta-. The gatekeeper also removes headers starting x-object-sysmeta- in anticipation of future support for system metadata being set for objects. This prevents clients from writing or reading system metadata. The required_filters list in swift/proxy/server.py is modified to include the gatekeeper middleware so that if the gatekeeper has not been configured in the pipeline then it will be automatically inserted close to the start of the pipeline. blueprint cluster-federation Change-Id: I80b8b14243cc59505f8c584920f8f527646b5f45 --- etc/proxy-server.conf-sample | 11 +- setup.cfg | 1 + swift/account/server.py | 5 +- swift/common/middleware/gatekeeper.py | 94 ++++++++++++++ swift/common/request_helpers.py | 106 ++++++++++++++++ swift/common/wsgi.py | 15 +++ swift/container/server.py | 10 +- swift/obj/server.py | 11 +- swift/proxy/controllers/base.py | 55 ++++++--- swift/proxy/controllers/obj.py | 3 +- swift/proxy/server.py | 17 ++- test/unit/account/test_server.py | 115 ++++++++++++++++++ .../unit/common/middleware/test_gatekeeper.py | 115 ++++++++++++++++++ test/unit/common/test_request_helpers.py | 70 +++++++++++ test/unit/common/test_wsgi.py | 105 +++++++++++++++- test/unit/container/test_server.py | 109 +++++++++++++++++ test/unit/proxy/controllers/test_account.py | 57 +++++++++ test/unit/proxy/controllers/test_base.py | 61 +++++++++- test/unit/proxy/controllers/test_container.py | 56 +++++++++ test/unit/proxy/test_server.py | 6 +- 20 files changed, 979 insertions(+), 43 deletions(-) create mode 100644 swift/common/middleware/gatekeeper.py create mode 100644 test/unit/common/middleware/test_gatekeeper.py create mode 100644 test/unit/common/test_request_helpers.py diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 1c1d4cc9d4..14945180d6 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -69,7 +69,7 @@ # eventlet_debug = false [pipeline:main] -pipeline = catch_errors healthcheck proxy-logging cache bulk slo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk slo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server [app:proxy-server] use = egg:swift#proxy @@ -509,3 +509,12 @@ use = egg:swift#slo [filter:account-quotas] use = egg:swift#account_quotas + +[filter:gatekeeper] +use = egg:swift#gatekeeper +# You can override the default log routing for this filter here: +# set log_name = gatekeeper +# set log_facility = LOG_LOCAL0 +# set log_level = INFO +# set log_headers = false +# set log_address = /dev/log diff --git a/setup.cfg b/setup.cfg index 02c115bbf8..1102b86502 100644 --- a/setup.cfg +++ b/setup.cfg @@ -86,6 +86,7 @@ paste.filter_factory = proxy_logging = swift.common.middleware.proxy_logging:filter_factory slo = swift.common.middleware.slo:filter_factory list_endpoints = swift.common.middleware.list_endpoints:filter_factory + gatekeeper = swift.common.middleware.gatekeeper:filter_factory [build_sphinx] all_files = 1 diff --git a/swift/account/server.py b/swift/account/server.py index 1c1ea0a4d2..8a0c12744d 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -37,6 +37,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, \ HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, \ HTTPPreconditionFailed, HTTPConflict, Request, \ HTTPInsufficientStorage, HTTPException +from swift.common.request_helpers import is_sys_or_user_meta DATADIR = 'accounts' @@ -152,7 +153,7 @@ class AccountController(object): metadata = {} metadata.update((key, (value, timestamp)) for key, value in req.headers.iteritems() - if key.lower().startswith('x-account-meta-')) + if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata) if created: @@ -258,7 +259,7 @@ class AccountController(object): metadata = {} metadata.update((key, (value, timestamp)) for key, value in req.headers.iteritems() - if key.lower().startswith('x-account-meta-')) + if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata) return HTTPNoContent(request=req) diff --git a/swift/common/middleware/gatekeeper.py b/swift/common/middleware/gatekeeper.py new file mode 100644 index 0000000000..4dc67e81cb --- /dev/null +++ b/swift/common/middleware/gatekeeper.py @@ -0,0 +1,94 @@ +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +The ``gatekeeper`` middleware imposes restrictions on the headers that +may be included with requests and responses. Request headers are filtered +to remove headers that should never be generated by a client. Similarly, +response headers are filtered to remove private headers that should +never be passed to a client. + +The ``gatekeeper`` middleware must always be present in the proxy server +wsgi pipeline. It should be configured close to the start of the pipeline +specified in ``/etc/swift/proxy-server.conf``, immediately after catch_errors +and before any other middleware. It is essential that it is configured ahead +of all middlewares using system metadata in order that they function +correctly. + +If ``gatekeeper`` middleware is not configured in the pipeline then it will be +automatically inserted close to the start of the pipeline by the proxy server. +""" + + +from swift.common.swob import wsgify +from swift.common.utils import get_logger +from swift.common.request_helpers import remove_items, get_sys_meta_prefix +import re + +""" +A list of python regular expressions that will be used to +match against inbound request headers. Matching headers will +be removed from the request. +""" +# Exclude headers starting with a sysmeta prefix. +# If adding to this list, note that these are regex patterns, +# so use a trailing $ to constrain to an exact header match +# rather than prefix match. +inbound_exclusions = [get_sys_meta_prefix('account'), + get_sys_meta_prefix('container'), + get_sys_meta_prefix('object')] +# 'x-object-sysmeta' is reserved in anticipation of future support +# for system metadata being applied to objects + + +""" +A list of python regular expressions that will be used to +match against outbound response headers. Matching headers will +be removed from the response. +""" +outbound_exclusions = inbound_exclusions + + +def make_exclusion_test(exclusions): + expr = '|'.join(exclusions) + test = re.compile(expr, re.IGNORECASE) + return test.match + + +class GatekeeperMiddleware(object): + def __init__(self, app, conf): + self.app = app + self.logger = get_logger(conf, log_route='gatekeeper') + self.inbound_condition = make_exclusion_test(inbound_exclusions) + self.outbound_condition = make_exclusion_test(outbound_exclusions) + + @wsgify + def __call__(self, req): + removed = remove_items(req.headers, self.inbound_condition) + if removed: + self.logger.debug('removed request headers: %s' % removed) + resp = req.get_response(self.app) + removed = remove_items(resp.headers, self.outbound_condition) + if removed: + self.logger.debug('removed response headers: %s' % removed) + return resp + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + + def gatekeeper_filter(app): + return GatekeeperMiddleware(app, conf) + return gatekeeper_filter diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py index c5584d5796..77672f1ce9 100644 --- a/swift/common/request_helpers.py +++ b/swift/common/request_helpers.py @@ -87,3 +87,109 @@ def split_and_validate_path(request, minsegs=1, maxsegs=None, except ValueError as err: raise HTTPBadRequest(body=str(err), request=request, content_type='text/plain') + + +def is_user_meta(server_type, key): + """ + Tests if a header key starts with and is longer than the user + metadata prefix for given server type. + + :param server_type: type of backend server i.e. [account|container|object] + :param key: header key + :returns: True if the key satisfies the test, False otherwise + """ + if len(key) <= 8 + len(server_type): + return False + return key.lower().startswith(get_user_meta_prefix(server_type)) + + +def is_sys_meta(server_type, key): + """ + Tests if a header key starts with and is longer than the system + metadata prefix for given server type. + + :param server_type: type of backend server i.e. [account|container|object] + :param key: header key + :returns: True if the key satisfies the test, False otherwise + """ + if len(key) <= 11 + len(server_type): + return False + return key.lower().startswith(get_sys_meta_prefix(server_type)) + + +def is_sys_or_user_meta(server_type, key): + """ + Tests if a header key starts with and is longer than the user or system + metadata prefix for given server type. + + :param server_type: type of backend server i.e. [account|container|object] + :param key: header key + :returns: True if the key satisfies the test, False otherwise + """ + return is_user_meta(server_type, key) or is_sys_meta(server_type, key) + + +def strip_user_meta_prefix(server_type, key): + """ + Removes the user metadata prefix for a given server type from the start + of a header key. + + :param server_type: type of backend server i.e. [account|container|object] + :param key: header key + :returns: stripped header key + """ + return key[len(get_user_meta_prefix(server_type)):] + + +def strip_sys_meta_prefix(server_type, key): + """ + Removes the system metadata prefix for a given server type from the start + of a header key. + + :param server_type: type of backend server i.e. [account|container|object] + :param key: header key + :returns: stripped header key + """ + return key[len(get_sys_meta_prefix(server_type)):] + + +def get_user_meta_prefix(server_type): + """ + Returns the prefix for user metadata headers for given server type. + + This prefix defines the namespace for headers that will be persisted + by backend servers. + + :param server_type: type of backend server i.e. [account|container|object] + :returns: prefix string for server type's user metadata headers + """ + return 'x-%s-%s-' % (server_type.lower(), 'meta') + + +def get_sys_meta_prefix(server_type): + """ + Returns the prefix for system metadata headers for given server type. + + This prefix defines the namespace for headers that will be persisted + by backend servers. + + :param server_type: type of backend server i.e. [account|container|object] + :returns: prefix string for server type's system metadata headers + """ + return 'x-%s-%s-' % (server_type.lower(), 'sysmeta') + + +def remove_items(headers, condition): + """ + Removes items from a dict whose keys satisfy + the given condition. + + :param headers: a dict of headers + :param condition: a function that will be passed the header key as a + single argument and should return True if the header is to be removed. + :returns: a dict, possibly empty, of headers that have been removed + """ + removed = {} + keys = filter(condition, headers) + removed.update((key, headers.pop(key)) for key in keys) + return removed diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index e61e1545c5..d2a75c6c0c 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -213,6 +213,21 @@ class PipelineWrapper(object): except ValueError: return False + def startswith(self, entry_point_name): + """ + Tests if the pipeline starts with the given entry point name. + + :param entry_point_name: entry point of middleware or app (Swift only) + + :returns: True if entry_point_name is first in pipeline, False + otherwise + """ + try: + first_ctx = self.context.filter_contexts[0] + except IndexError: + first_ctx = self.context.app_context + return first_ctx.entry_point_name == entry_point_name + def _format_for_display(self, ctx): if ctx.entry_point_name: return ctx.entry_point_name diff --git a/swift/container/server.py b/swift/container/server.py index f177f4ebfa..5ae4e35f07 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -26,7 +26,7 @@ import swift.common.db from swift.container.backend import ContainerBroker from swift.common.db import DatabaseAlreadyExists from swift.common.request_helpers import get_param, get_listing_content_type, \ - split_and_validate_path + split_and_validate_path, is_sys_or_user_meta from swift.common.utils import get_logger, hash_path, public, \ normalize_timestamp, storage_directory, validate_sync_to, \ config_true_value, json, timing_stats, replication, \ @@ -266,7 +266,7 @@ class ContainerController(object): (key, (value, timestamp)) for key, value in req.headers.iteritems() if key.lower() in self.save_headers or - key.lower().startswith('x-container-meta-')) + is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ @@ -307,7 +307,7 @@ class ContainerController(object): (key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and (key.lower() in self.save_headers or - key.lower().startswith('x-container-meta-'))) + is_sys_or_user_meta('container', key))) headers['Content-Type'] = out_content_type return HTTPNoContent(request=req, headers=headers, charset='utf-8') @@ -374,7 +374,7 @@ class ContainerController(object): } for key, (value, timestamp) in broker.metadata.iteritems(): if value and (key.lower() in self.save_headers or - key.lower().startswith('x-container-meta-')): + is_sys_or_user_meta('container', key)): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') @@ -452,7 +452,7 @@ class ContainerController(object): metadata.update( (key, (value, timestamp)) for key, value in req.headers.iteritems() if key.lower() in self.save_headers or - key.lower().startswith('x-container-meta-')) + is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ diff --git a/swift/obj/server.py b/swift/obj/server.py index b22b8221f8..99a5db89cf 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -38,7 +38,7 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \ DiskFileDeviceUnavailable, DiskFileExpired from swift.obj import ssync_receiver from swift.common.http import is_success -from swift.common.request_helpers import split_and_validate_path +from swift.common.request_helpers import split_and_validate_path, is_user_meta from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ HTTPInternalServerError, HTTPNoContent, HTTPNotFound, HTTPNotModified, \ HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \ @@ -338,7 +338,7 @@ class ObjectController(object): return HTTPConflict(request=request) metadata = {'X-Timestamp': request.headers['x-timestamp']} metadata.update(val for val in request.headers.iteritems() - if val[0].startswith('X-Object-Meta-')) + if is_user_meta('object', val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() @@ -422,8 +422,7 @@ class ObjectController(object): 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.iteritems() - if val[0].lower().startswith('x-object-meta-') - and len(val[0]) > 14) + if is_user_meta('object', val[0])) for header_key in ( request.headers.get('X-Backend-Replication-Headers') or self.allowed_headers): @@ -504,7 +503,7 @@ class ObjectController(object): response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): - if key.lower().startswith('x-object-meta-') or \ + if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] @@ -545,7 +544,7 @@ class ObjectController(object): response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): - if key.lower().startswith('x-object-meta-') or \ + if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 0f76950836..5637841a29 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -48,6 +48,8 @@ from swift.common.http import is_informational, is_success, is_redirection, \ HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED from swift.common.swob import Request, Response, HeaderKeyDict, Range, \ HTTPException, HTTPRequestedRangeNotSatisfiable +from swift.common.request_helpers import strip_sys_meta_prefix, \ + strip_user_meta_prefix, is_user_meta, is_sys_meta, is_sys_or_user_meta def update_headers(response, headers): @@ -106,11 +108,32 @@ def get_container_memcache_key(account, container): return cache_key +def _prep_headers_to_info(headers, server_type): + """ + Helper method that iterates once over a dict of headers, + converting all keys to lower case and separating + into subsets containing user metadata, system metadata + and other headers. + """ + meta = {} + sysmeta = {} + other = {} + for key, val in dict(headers).iteritems(): + lkey = key.lower() + if is_user_meta(server_type, lkey): + meta[strip_user_meta_prefix(server_type, lkey)] = val + elif is_sys_meta(server_type, lkey): + sysmeta[strip_sys_meta_prefix(server_type, lkey)] = val + else: + other[lkey] = val + return other, meta, sysmeta + + def headers_to_account_info(headers, status_int=HTTP_OK): """ Construct a cacheable dict of account info based on response headers. """ - headers = dict((k.lower(), v) for k, v in dict(headers).iteritems()) + headers, meta, sysmeta = _prep_headers_to_info(headers, 'account') return { 'status': status_int, # 'container_count' anomaly: @@ -120,9 +143,8 @@ def headers_to_account_info(headers, status_int=HTTP_OK): 'container_count': headers.get('x-account-container-count'), 'total_object_count': headers.get('x-account-object-count'), 'bytes': headers.get('x-account-bytes-used'), - 'meta': dict((key[15:], value) - for key, value in headers.iteritems() - if key.startswith('x-account-meta-')) + 'meta': meta, + 'sysmeta': sysmeta } @@ -130,7 +152,7 @@ def headers_to_container_info(headers, status_int=HTTP_OK): """ Construct a cacheable dict of container info based on response headers. """ - headers = dict((k.lower(), v) for k, v in dict(headers).iteritems()) + headers, meta, sysmeta = _prep_headers_to_info(headers, 'container') return { 'status': status_int, 'read_acl': headers.get('x-container-read'), @@ -140,16 +162,12 @@ def headers_to_container_info(headers, status_int=HTTP_OK): 'bytes': headers.get('x-container-bytes-used'), 'versions': headers.get('x-versions-location'), 'cors': { - 'allow_origin': headers.get( - 'x-container-meta-access-control-allow-origin'), - 'expose_headers': headers.get( - 'x-container-meta-access-control-expose-headers'), - 'max_age': headers.get( - 'x-container-meta-access-control-max-age') + 'allow_origin': meta.get('access-control-allow-origin'), + 'expose_headers': meta.get('access-control-expose-headers'), + 'max_age': meta.get('access-control-max-age') }, - 'meta': dict((key[17:], value) - for key, value in headers.iteritems() - if key.startswith('x-container-meta-')) + 'meta': meta, + 'sysmeta': sysmeta } @@ -157,14 +175,12 @@ def headers_to_object_info(headers, status_int=HTTP_OK): """ Construct a cacheable dict of object info based on response headers. """ - headers = dict((k.lower(), v) for k, v in dict(headers).iteritems()) + headers, meta, sysmeta = _prep_headers_to_info(headers, 'object') info = {'status': status_int, 'length': headers.get('content-length'), 'type': headers.get('content-type'), 'etag': headers.get('etag'), - 'meta': dict((key[14:], value) - for key, value in headers.iteritems() - if key.startswith('x-object-meta-')) + 'meta': meta } return info @@ -854,11 +870,10 @@ class Controller(object): if k.lower().startswith(x_remove) or k.lower() in self._x_remove_headers()) - x_meta = 'x-%s-meta-' % st dst_headers.update((k.lower(), v) for k, v in src_headers.iteritems() if k.lower() in self.pass_through_headers or - k.lower().startswith(x_meta)) + is_sys_or_user_meta(st, k)) def generate_request_headers(self, orig_req=None, additional=None, transfer=False): diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 189e909756..603c4f6165 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -59,6 +59,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \ HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \ HTTPServerError, HTTPServiceUnavailable, Request, Response, \ HTTPClientDisconnect, HTTPNotImplemented, HTTPException +from swift.common.request_helpers import is_user_meta def segment_listing_iter(listing): @@ -78,7 +79,7 @@ def copy_headers_into(from_r, to_r): """ pass_headers = ['x-delete-at'] for k, v in from_r.headers.items(): - if k.lower().startswith('x-object-meta-') or k.lower() in pass_headers: + if is_user_meta('object', k) or k.lower() in pass_headers: to_r.headers[k] = v diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 1d5f968d3c..2a1f0e5850 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -51,7 +51,17 @@ from swift.common.swob import HTTPBadRequest, HTTPForbidden, \ # example, 'after: ["catch_errors", "bulk"]' would install this middleware # after catch_errors and bulk if both were present, but if bulk were absent, # would just install it after catch_errors. -required_filters = [{'name': 'catch_errors'}] +# +# "after_fn" (optional) a function that takes a PipelineWrapper object as its +# single argument and returns a list of middlewares that this middleware should +# come after. This list overrides any defined by the "after" field. +required_filters = [ + {'name': 'catch_errors'}, + {'name': 'gatekeeper', + 'after_fn': lambda pipe: (['catch_errors'] + if pipe.startswith("catch_errors") + else [])} +] class Application(object): @@ -505,7 +515,10 @@ class Application(object): for filter_spec in reversed(required_filters): filter_name = filter_spec['name'] if filter_name not in pipe: - afters = filter_spec.get('after', []) + if 'after_fn' in filter_spec: + afters = filter_spec['after_fn'](pipe) + else: + afters = filter_spec.get('after', []) insert_at = 0 for after in afters: try: diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index 748c3173c6..c91ff0d416 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -26,6 +26,7 @@ import xml.dom.minidom from swift.common.swob import Request from swift.account.server import AccountController, ACCOUNT_LISTING_LIMIT from swift.common.utils import normalize_timestamp, replication, public +from swift.common.request_helpers import get_sys_meta_prefix class TestAccountController(unittest.TestCase): @@ -371,6 +372,67 @@ class TestAccountController(unittest.TestCase): self.assertEqual(resp.status_int, 204) self.assert_('x-account-meta-test' not in resp.headers) + def test_PUT_GET_sys_metadata(self): + prefix = get_sys_meta_prefix('account') + hdr = '%stest' % prefix + hdr2 = '%stest2' % prefix + # Set metadata header + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(1), + hdr.title(): 'Value'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'Value') + # Set another metadata header, ensuring old one doesn't disappear + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(1), + hdr2.title(): 'Value2'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'Value') + self.assertEqual(resp.headers.get(hdr2), 'Value2') + # Update metadata header + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(3), + hdr.title(): 'New Value'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 202) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'New Value') + # Send old update to metadata header + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(2), + hdr.title(): 'Old Value'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 202) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'New Value') + # Remove metadata header (by setting it to empty) + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(4), + hdr.title(): ''}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 202) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assert_(hdr not in resp.headers) + def test_PUT_invalid_partition(self): req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) @@ -435,6 +497,59 @@ class TestAccountController(unittest.TestCase): self.assertEqual(resp.status_int, 204) self.assert_('x-account-meta-test' not in resp.headers) + def test_POST_HEAD_sys_metadata(self): + prefix = get_sys_meta_prefix('account') + hdr = '%stest' % prefix + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(1)}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + # Set metadata header + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(1), + hdr.title(): 'Value'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'Value') + # Update metadata header + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(3), + hdr.title(): 'New Value'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'New Value') + # Send old update to metadata header + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(2), + hdr.title(): 'Old Value'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(resp.headers.get(hdr), 'New Value') + # Remove metadata header (by setting it to empty) + req = Request.blank( + '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(4), + hdr.title(): ''}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assert_(hdr not in resp.headers) + def test_POST_invalid_partition(self): req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_TIMESTAMP': '1'}) diff --git a/test/unit/common/middleware/test_gatekeeper.py b/test/unit/common/middleware/test_gatekeeper.py new file mode 100644 index 0000000000..294273171a --- /dev/null +++ b/test/unit/common/middleware/test_gatekeeper.py @@ -0,0 +1,115 @@ +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from swift.common.swob import Request, Response +from swift.common.middleware import gatekeeper + + +class FakeApp(object): + def __init__(self, headers={}): + self.headers = headers + self.req = None + + def __call__(self, env, start_response): + self.req = Request(env) + return Response(request=self.req, body='FAKE APP', + headers=self.headers)(env, start_response) + + +class TestGatekeeper(unittest.TestCase): + methods = ['PUT', 'POST', 'GET', 'DELETE', 'HEAD', 'COPY', 'OPTIONS'] + + allowed_headers = {'xx-account-sysmeta-foo': 'value', + 'xx-container-sysmeta-foo': 'value', + 'xx-object-sysmeta-foo': 'value', + 'x-account-meta-foo': 'value', + 'x-container-meta-foo': 'value', + 'x-object-meta-foo': 'value', + 'x-timestamp-foo': 'value'} + + sysmeta_headers = {'x-account-sysmeta-': 'value', + 'x-container-sysmeta-': 'value', + 'x-object-sysmeta-': 'value', + 'x-account-sysmeta-foo': 'value', + 'x-container-sysmeta-foo': 'value', + 'x-object-sysmeta-foo': 'value', + 'X-Account-Sysmeta-BAR': 'value', + 'X-Container-Sysmeta-BAR': 'value', + 'X-Object-Sysmeta-BAR': 'value'} + + forbidden_headers_out = dict(sysmeta_headers) + forbidden_headers_in = dict(sysmeta_headers) + + def _assertHeadersEqual(self, expected, actual): + for key in expected: + self.assertTrue(key.lower() in actual, + '%s missing from %s' % (key, actual)) + + def _assertHeadersAbsent(self, unexpected, actual): + for key in unexpected: + self.assertTrue(key.lower() not in actual, + '%s is in %s' % (key, actual)) + + def get_app(self, app, global_conf, **local_conf): + factory = gatekeeper.filter_factory(global_conf, **local_conf) + return factory(app) + + def test_ok_header(self): + req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers=self.allowed_headers) + fake_app = FakeApp() + app = self.get_app(fake_app, {}) + resp = req.get_response(app) + self.assertEquals('200 OK', resp.status) + self.assertEquals(resp.body, 'FAKE APP') + self._assertHeadersEqual(self.allowed_headers, fake_app.req.headers) + + def _test_reserved_header_removed_inbound(self, method): + headers = dict(self.forbidden_headers_in) + headers.update(self.allowed_headers) + req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': method}, + headers=headers) + fake_app = FakeApp() + app = self.get_app(fake_app, {}) + resp = req.get_response(app) + self.assertEquals('200 OK', resp.status) + self._assertHeadersEqual(self.allowed_headers, fake_app.req.headers) + self._assertHeadersAbsent(self.forbidden_headers_in, + fake_app.req.headers) + + def test_reserved_header_removed_inbound(self): + for method in self.methods: + self._test_reserved_header_removed_inbound(method) + + def _test_reserved_header_removed_outbound(self, method): + headers = dict(self.forbidden_headers_out) + headers.update(self.allowed_headers) + req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': method}) + fake_app = FakeApp(headers=headers) + app = self.get_app(fake_app, {}) + resp = req.get_response(app) + self.assertEquals('200 OK', resp.status) + self._assertHeadersEqual(self.allowed_headers, resp.headers) + self._assertHeadersAbsent(self.forbidden_headers_out, resp.headers) + + def test_reserved_header_removed_outbound(self): + for method in self.methods: + self._test_reserved_header_removed_outbound(method) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_request_helpers.py b/test/unit/common/test_request_helpers.py new file mode 100644 index 0000000000..8bb382db1d --- /dev/null +++ b/test/unit/common/test_request_helpers.py @@ -0,0 +1,70 @@ +# Copyright (c) 2010-2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for swift.common.request_helpers""" + +import unittest +from swift.common.request_helpers import is_sys_meta, is_user_meta, \ + is_sys_or_user_meta, strip_sys_meta_prefix, strip_user_meta_prefix, \ + remove_items + +server_types = ['account', 'container', 'object'] + + +class TestRequestHelpers(unittest.TestCase): + def test_is_user_meta(self): + m_type = 'meta' + for st in server_types: + self.assertTrue(is_user_meta(st, 'x-%s-%s-foo' % (st, m_type))) + self.assertFalse(is_user_meta(st, 'x-%s-%s-' % (st, m_type))) + self.assertFalse(is_user_meta(st, 'x-%s-%sfoo' % (st, m_type))) + + def test_is_sys_meta(self): + m_type = 'sysmeta' + for st in server_types: + self.assertTrue(is_sys_meta(st, 'x-%s-%s-foo' % (st, m_type))) + self.assertFalse(is_sys_meta(st, 'x-%s-%s-' % (st, m_type))) + self.assertFalse(is_sys_meta(st, 'x-%s-%sfoo' % (st, m_type))) + + def test_is_sys_or_user_meta(self): + m_types = ['sysmeta', 'meta'] + for mt in m_types: + for st in server_types: + self.assertTrue(is_sys_or_user_meta(st, 'x-%s-%s-foo' + % (st, mt))) + self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%s-' + % (st, mt))) + self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%sfoo' + % (st, mt))) + + def test_strip_sys_meta_prefix(self): + mt = 'sysmeta' + for st in server_types: + self.assertEquals(strip_sys_meta_prefix(st, 'x-%s-%s-a' + % (st, mt)), 'a') + + def test_strip_user_meta_prefix(self): + mt = 'meta' + for st in server_types: + self.assertEquals(strip_user_meta_prefix(st, 'x-%s-%s-a' + % (st, mt)), 'a') + + def test_remove_items(self): + src = {'a': 'b', + 'c': 'd'} + test = lambda x: x == 'a' + rem = remove_items(src, test) + self.assertEquals(src, {'c': 'd'}) + self.assertEquals(rem, {'a': 'b'}) diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py index d1e8f5130c..6172c3fda9 100644 --- a/test/unit/common/test_wsgi.py +++ b/test/unit/common/test_wsgi.py @@ -35,6 +35,7 @@ from eventlet import listen import mock import swift.common.middleware.catch_errors +import swift.common.middleware.gatekeeper import swift.proxy.server from swift.common.swob import Request @@ -143,6 +144,9 @@ class TestWSGI(unittest.TestCase): # verify pipeline is catch_errors -> proxy-server expected = swift.common.middleware.catch_errors.CatchErrorMiddleware self.assert_(isinstance(app, expected)) + app = app.app + expected = swift.common.middleware.gatekeeper.GatekeeperMiddleware + self.assert_(isinstance(app, expected)) self.assert_(isinstance(app.app, swift.proxy.server.Application)) # config settings applied to app instance self.assertEquals(0.2, app.app.conn_timeout) @@ -706,6 +710,31 @@ class TestPipelineWrapper(unittest.TestCase): # filters in the pipeline. return [c.entry_point_name for c in self.pipe.context.filter_contexts] + def test_startswith(self): + self.assertTrue(self.pipe.startswith("healthcheck")) + self.assertFalse(self.pipe.startswith("tempurl")) + + def test_startswith_no_filters(self): + config = """ + [DEFAULT] + swift_dir = TEMPDIR + + [pipeline:main] + pipeline = proxy-server + + [app:proxy-server] + use = egg:swift#proxy + conn_timeout = 0.2 + """ + contents = dedent(config) + with temptree(['proxy-server.conf']) as t: + conf_file = os.path.join(t, 'proxy-server.conf') + with open(conf_file, 'w') as f: + f.write(contents.replace('TEMPDIR', t)) + ctx = wsgi.loadcontext(loadwsgi.APP, conf_file, global_conf={}) + pipe = wsgi.PipelineWrapper(ctx) + self.assertTrue(pipe.startswith('proxy')) + def test_insert_filter(self): original_modules = ['healthcheck', 'catch_errors', None] self.assertEqual(self._entry_point_names(), original_modules) @@ -789,7 +818,7 @@ class TestPipelineModification(unittest.TestCase): swift_dir = TEMPDIR [pipeline:main] - pipeline = catch_errors proxy-server + pipeline = catch_errors gatekeeper proxy-server [app:proxy-server] use = egg:swift#proxy @@ -797,6 +826,9 @@ class TestPipelineModification(unittest.TestCase): [filter:catch_errors] use = egg:swift#catch_errors + + [filter:gatekeeper] + use = egg:swift#gatekeeper """ contents = dedent(config) @@ -809,6 +841,7 @@ class TestPipelineModification(unittest.TestCase): self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', + 'swift.common.middleware.gatekeeper', 'swift.proxy.server']) def test_proxy_modify_wsgi_pipeline(self): @@ -835,8 +868,11 @@ class TestPipelineModification(unittest.TestCase): _fake_rings(t) app = wsgi.loadapp(conf_file, global_conf={}) - self.assertEqual(self.pipeline_modules(app)[0], - 'swift.common.middleware.catch_errors') + self.assertEqual(self.pipeline_modules(app), + ['swift.common.middleware.catch_errors', + 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.healthcheck', + 'swift.proxy.server']) def test_proxy_modify_wsgi_pipeline_ordering(self): config = """ @@ -892,6 +928,69 @@ class TestPipelineModification(unittest.TestCase): 'swift.common.middleware.tempurl', 'swift.proxy.server']) + def _proxy_modify_wsgi_pipeline(self, pipe): + config = """ + [DEFAULT] + swift_dir = TEMPDIR + + [pipeline:main] + pipeline = %s + + [app:proxy-server] + use = egg:swift#proxy + conn_timeout = 0.2 + + [filter:healthcheck] + use = egg:swift#healthcheck + + [filter:catch_errors] + use = egg:swift#catch_errors + + [filter:gatekeeper] + use = egg:swift#gatekeeper + """ + config = config % (pipe,) + contents = dedent(config) + with temptree(['proxy-server.conf']) as t: + conf_file = os.path.join(t, 'proxy-server.conf') + with open(conf_file, 'w') as f: + f.write(contents.replace('TEMPDIR', t)) + _fake_rings(t) + app = wsgi.loadapp(conf_file, global_conf={}) + return app + + def test_gatekeeper_insertion_catch_errors_configured_at_start(self): + # catch_errors is configured at start, gatekeeper is not configured, + # so gatekeeper should be inserted just after catch_errors + pipe = 'catch_errors healthcheck proxy-server' + app = self._proxy_modify_wsgi_pipeline(pipe) + self.assertEqual(self.pipeline_modules(app), [ + 'swift.common.middleware.catch_errors', + 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.healthcheck', + 'swift.proxy.server']) + + def test_gatekeeper_insertion_catch_errors_configured_not_at_start(self): + # catch_errors is configured, gatekeeper is not configured, so + # gatekeeper should be inserted at start of pipeline + pipe = 'healthcheck catch_errors proxy-server' + app = self._proxy_modify_wsgi_pipeline(pipe) + self.assertEqual(self.pipeline_modules(app), [ + 'swift.common.middleware.gatekeeper', + 'swift.common.middleware.healthcheck', + 'swift.common.middleware.catch_errors', + 'swift.proxy.server']) + + def test_catch_errors_gatekeeper_configured_not_at_start(self): + # catch_errors is configured, gatekeeper is configured, so + # no change should be made to pipeline + pipe = 'healthcheck catch_errors gatekeeper proxy-server' + app = self._proxy_modify_wsgi_pipeline(pipe) + self.assertEqual(self.pipeline_modules(app), [ + 'swift.common.middleware.healthcheck', + 'swift.common.middleware.catch_errors', + 'swift.common.middleware.gatekeeper', + 'swift.proxy.server']) if __name__ == '__main__': unittest.main() diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 7662006fec..a65e5d83ad 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -31,6 +31,7 @@ import swift.container from swift.container import server as container_server from swift.common.utils import normalize_timestamp, mkdirs, public, replication from test.unit import fake_http_connect +from swift.common.request_helpers import get_sys_meta_prefix @contextmanager @@ -292,6 +293,64 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 204) self.assert_('x-container-meta-test' not in resp.headers) + def test_PUT_GET_sys_metadata(self): + prefix = get_sys_meta_prefix('container') + key = '%sTest' % prefix + key2 = '%sTest2' % prefix + # Set metadata header + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(1), + key: 'Value'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c') + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), 'Value') + # Set another metadata header, ensuring old one doesn't disappear + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(1), + key2: 'Value2'}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c') + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), 'Value') + self.assertEquals(resp.headers.get(key2.lower()), 'Value2') + # Update metadata header + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(3), + key: 'New Value'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 202) + req = Request.blank('/sda1/p/a/c') + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), + 'New Value') + # Send old update to metadata header + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(2), + key: 'Old Value'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 202) + req = Request.blank('/sda1/p/a/c') + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), + 'New Value') + # Remove metadata header (by setting it to empty) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(4), + key: ''}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 202) + req = Request.blank('/sda1/p/a/c') + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 204) + self.assert_(key.lower() not in resp.headers) + def test_PUT_invalid_partition(self): req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) @@ -369,6 +428,56 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 204) self.assert_('x-container-meta-test' not in resp.headers) + def test_POST_HEAD_sys_metadata(self): + prefix = get_sys_meta_prefix('container') + key = '%sTest' % prefix + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': normalize_timestamp(1)}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + # Set metadata header + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(1), + key: 'Value'}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), 'Value') + # Update metadata header + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(3), + key: 'New Value'}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), + 'New Value') + # Send old update to metadata header + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(2), + key: 'Old Value'}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 204) + self.assertEquals(resp.headers.get(key.lower()), + 'New Value') + # Remove metadata header (by setting it to empty) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': normalize_timestamp(4), + key: ''}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 204) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 204) + self.assert_(key.lower() not in resp.headers) + def test_POST_invalid_partition(self): req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_TIMESTAMP': '1'}) diff --git a/test/unit/proxy/controllers/test_account.py b/test/unit/proxy/controllers/test_account.py index 29ed09bd62..eefd57dd28 100644 --- a/test/unit/proxy/controllers/test_account.py +++ b/test/unit/proxy/controllers/test_account.py @@ -21,6 +21,7 @@ from swift.proxy import server as proxy_server from swift.proxy.controllers.base import headers_to_account_info from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH as MAX_ANAME_LEN from test.unit import fake_http_connect, FakeRing, FakeMemcache +from swift.common.request_helpers import get_sys_meta_prefix class TestAccountController(unittest.TestCase): @@ -95,6 +96,62 @@ class TestAccountController(unittest.TestCase): resp = controller.POST(req) self.assertEquals(400, resp.status_int) + def _make_callback_func(self, context): + def callback(ipaddr, port, device, partition, method, path, + headers=None, query_string=None, ssl=False): + context['method'] = method + context['path'] = path + context['headers'] = headers or {} + return callback + + def test_sys_meta_headers_PUT(self): + # check that headers in sys meta namespace make it through + # the proxy controller + sys_meta_key = '%stest' % get_sys_meta_prefix('account') + sys_meta_key = sys_meta_key.title() + user_meta_key = 'X-Account-Meta-Test' + # allow PUTs to account... + self.app.allow_account_management = True + controller = proxy_server.AccountController(self.app, 'a') + context = {} + callback = self._make_callback_func(context) + hdrs_in = {sys_meta_key: 'foo', + user_meta_key: 'bar', + 'x-timestamp': '1.0'} + req = Request.blank('/v1/a', headers=hdrs_in) + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, give_connect=callback)): + controller.PUT(req) + self.assertEqual(context['method'], 'PUT') + self.assertTrue(sys_meta_key in context['headers']) + self.assertEqual(context['headers'][sys_meta_key], 'foo') + self.assertTrue(user_meta_key in context['headers']) + self.assertEqual(context['headers'][user_meta_key], 'bar') + self.assertNotEqual(context['headers']['x-timestamp'], '1.0') + + def test_sys_meta_headers_POST(self): + # check that headers in sys meta namespace make it through + # the proxy controller + sys_meta_key = '%stest' % get_sys_meta_prefix('account') + sys_meta_key = sys_meta_key.title() + user_meta_key = 'X-Account-Meta-Test' + controller = proxy_server.AccountController(self.app, 'a') + context = {} + callback = self._make_callback_func(context) + hdrs_in = {sys_meta_key: 'foo', + user_meta_key: 'bar', + 'x-timestamp': '1.0'} + req = Request.blank('/v1/a', headers=hdrs_in) + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, give_connect=callback)): + controller.POST(req) + self.assertEqual(context['method'], 'POST') + self.assertTrue(sys_meta_key in context['headers']) + self.assertEqual(context['headers'][sys_meta_key], 'foo') + self.assertTrue(user_meta_key in context['headers']) + self.assertEqual(context['headers'][user_meta_key], 'bar') + self.assertNotEqual(context['headers']['x-timestamp'], '1.0') + if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/controllers/test_base.py b/test/unit/proxy/controllers/test_base.py index e72eedcf32..0c94f90171 100644 --- a/test/unit/proxy/controllers/test_base.py +++ b/test/unit/proxy/controllers/test_base.py @@ -20,10 +20,11 @@ from swift.proxy.controllers.base import headers_to_container_info, \ get_container_memcache_key, get_account_info, get_account_memcache_key, \ get_object_env_key, _get_cache_key, get_info, get_object_info, \ Controller, GetOrHeadHandler -from swift.common.swob import Request, HTTPException +from swift.common.swob import Request, HTTPException, HeaderKeyDict from swift.common.utils import split_path from test.unit import fake_http_connect, FakeRing, FakeMemcache from swift.proxy import server as proxy_server +from swift.common.request_helpers import get_sys_meta_prefix FakeResponse_status_int = 201 @@ -365,6 +366,15 @@ class TestFuncs(unittest.TestCase): self.assertEquals(resp['meta']['whatevs'], 14) self.assertEquals(resp['meta']['somethingelse'], 0) + def test_headers_to_container_info_sys_meta(self): + prefix = get_sys_meta_prefix('container') + headers = {'%sWhatevs' % prefix: 14, + '%ssomethingelse' % prefix: 0} + resp = headers_to_container_info(headers.items(), 200) + self.assertEquals(len(resp['sysmeta']), 2) + self.assertEquals(resp['sysmeta']['whatevs'], 14) + self.assertEquals(resp['sysmeta']['somethingelse'], 0) + def test_headers_to_container_info_values(self): headers = { 'x-container-read': 'readvalue', @@ -396,6 +406,15 @@ class TestFuncs(unittest.TestCase): self.assertEquals(resp['meta']['whatevs'], 14) self.assertEquals(resp['meta']['somethingelse'], 0) + def test_headers_to_account_info_sys_meta(self): + prefix = get_sys_meta_prefix('account') + headers = {'%sWhatevs' % prefix: 14, + '%ssomethingelse' % prefix: 0} + resp = headers_to_account_info(headers.items(), 200) + self.assertEquals(len(resp['sysmeta']), 2) + self.assertEquals(resp['sysmeta']['whatevs'], 14) + self.assertEquals(resp['sysmeta']['somethingelse'], 0) + def test_headers_to_account_info_values(self): headers = { 'x-account-object-count': '10', @@ -473,3 +492,43 @@ class TestFuncs(unittest.TestCase): {'Range': 'bytes=-100'}) handler.fast_forward(20) self.assertEquals(handler.backend_headers['Range'], 'bytes=-80') + + def test_transfer_headers_with_sysmeta(self): + base = Controller(self.app) + good_hdrs = {'x-base-sysmeta-foo': 'ok', + 'X-Base-sysmeta-Bar': 'also ok'} + bad_hdrs = {'x-base-sysmeta-': 'too short'} + hdrs = dict(good_hdrs) + hdrs.update(bad_hdrs) + dst_hdrs = HeaderKeyDict() + base.transfer_headers(hdrs, dst_hdrs) + self.assertEqual(HeaderKeyDict(good_hdrs), dst_hdrs) + + def test_generate_request_headers(self): + base = Controller(self.app) + src_headers = {'x-remove-base-meta-owner': 'x', + 'x-base-meta-size': '151M', + 'new-owner': 'Kun'} + req = Request.blank('/v1/a/c/o', headers=src_headers) + dst_headers = base.generate_request_headers(req, transfer=True) + expected_headers = {'x-base-meta-owner': '', + 'x-base-meta-size': '151M'} + for k, v in expected_headers.iteritems(): + self.assertTrue(k in dst_headers) + self.assertEqual(v, dst_headers[k]) + self.assertFalse('new-owner' in dst_headers) + + def test_generate_request_headers_with_sysmeta(self): + base = Controller(self.app) + good_hdrs = {'x-base-sysmeta-foo': 'ok', + 'X-Base-sysmeta-Bar': 'also ok'} + bad_hdrs = {'x-base-sysmeta-': 'too short'} + hdrs = dict(good_hdrs) + hdrs.update(bad_hdrs) + req = Request.blank('/v1/a/c/o', headers=hdrs) + dst_headers = base.generate_request_headers(req, transfer=True) + for k, v in good_hdrs.iteritems(): + self.assertTrue(k.lower() in dst_headers) + self.assertEqual(v, dst_headers[k.lower()]) + for k, v in bad_hdrs.iteritems(): + self.assertFalse(k.lower() in dst_headers) diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py index e98e04fe8d..7c8ecf7075 100644 --- a/test/unit/proxy/controllers/test_container.py +++ b/test/unit/proxy/controllers/test_container.py @@ -20,6 +20,7 @@ from swift.common.swob import Request from swift.proxy import server as proxy_server from swift.proxy.controllers.base import headers_to_container_info from test.unit import fake_http_connect, FakeRing, FakeMemcache +from swift.common.request_helpers import get_sys_meta_prefix class TestContainerController(unittest.TestCase): @@ -62,6 +63,61 @@ class TestContainerController(unittest.TestCase): for key in owner_headers: self.assertTrue(key in resp.headers) + def _make_callback_func(self, context): + def callback(ipaddr, port, device, partition, method, path, + headers=None, query_string=None, ssl=False): + context['method'] = method + context['path'] = path + context['headers'] = headers or {} + return callback + + def test_sys_meta_headers_PUT(self): + # check that headers in sys meta namespace make it through + # the container controller + sys_meta_key = '%stest' % get_sys_meta_prefix('container') + sys_meta_key = sys_meta_key.title() + user_meta_key = 'X-Container-Meta-Test' + controller = proxy_server.ContainerController(self.app, 'a', 'c') + + context = {} + callback = self._make_callback_func(context) + hdrs_in = {sys_meta_key: 'foo', + user_meta_key: 'bar', + 'x-timestamp': '1.0'} + req = Request.blank('/v1/a/c', headers=hdrs_in) + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, give_connect=callback)): + controller.PUT(req) + self.assertEqual(context['method'], 'PUT') + self.assertTrue(sys_meta_key in context['headers']) + self.assertEqual(context['headers'][sys_meta_key], 'foo') + self.assertTrue(user_meta_key in context['headers']) + self.assertEqual(context['headers'][user_meta_key], 'bar') + self.assertNotEqual(context['headers']['x-timestamp'], '1.0') + + def test_sys_meta_headers_POST(self): + # check that headers in sys meta namespace make it through + # the container controller + sys_meta_key = '%stest' % get_sys_meta_prefix('container') + sys_meta_key = sys_meta_key.title() + user_meta_key = 'X-Container-Meta-Test' + controller = proxy_server.ContainerController(self.app, 'a', 'c') + context = {} + callback = self._make_callback_func(context) + hdrs_in = {sys_meta_key: 'foo', + user_meta_key: 'bar', + 'x-timestamp': '1.0'} + req = Request.blank('/v1/a/c', headers=hdrs_in) + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, give_connect=callback)): + controller.POST(req) + self.assertEqual(context['method'], 'POST') + self.assertTrue(sys_meta_key in context['headers']) + self.assertEqual(context['headers'][sys_meta_key], 'foo') + self.assertTrue(user_meta_key in context['headers']) + self.assertEqual(context['headers'][user_meta_key], 'bar') + self.assertNotEqual(context['headers']['x-timestamp'], '1.0') + if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 8803e4a797..80a13155bd 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -340,7 +340,8 @@ class TestController(unittest.TestCase): 'container_count': '12345', 'total_object_count': None, 'bytes': None, - 'meta': {}} + 'meta': {}, + 'sysmeta': {}} self.assertEquals(container_info, self.memcache.get(cache_key)) @@ -366,7 +367,8 @@ class TestController(unittest.TestCase): 'container_count': None, # internally keep None 'total_object_count': None, 'bytes': None, - 'meta': {}} + 'meta': {}, + 'sysmeta': {}} self.assertEquals(account_info, self.memcache.get(cache_key))