openstack
/
swift
Code Issues Proposed changes

Generic means for persisting system metadata. 

Middleware or core features may need to store metadata
against accounts or containers. This patch adds a
generic mechanism for system metadata to be persisted
in backend databases, without polluting the user
metadata namespace, by using the reserved header
namespace x-<server_type>-sysmeta-*.

Modifications are firstly that backend servers persist
system metadata headers alongside user metadata and
other system state.

For accounts and containers, system metadata in PUT
and POST requests is treated in a similar way to user
metadata. System metadata is not yet supported for
object requests.

Secondly, changes in the proxy controllers ensure that
headers in the system metadata namespace will pass through
in requests to backend servers.

Thirdly, system metadata returned from backend servers
in GET or HEAD responses is added to the cached info
dict, which middleware can access.

Finally, a gatekeeper middleware module is provided
which filters all system metadata headers from requests
and responses by removing headers with names starting
x-account-sysmeta-, x-container-sysmeta-. The gatekeeper
also removes headers starting x-object-sysmeta- in
anticipation of future support for system metadata being
set for objects. This prevents clients from writing or
reading system metadata.

The required_filters list in swift/proxy/server.py is
modified to include the gatekeeper middleware so that
if the gatekeeper has not been configured in the
pipeline then it will be automatically inserted close
to the start of the pipeline.

blueprint cluster-federation

Change-Id: I80b8b14243cc59505f8c584920f8f527646b5f45
This commit is contained in:
anc 2013-12-03 22:02:39 +00:00
parent 71d52d300c
commit 6164fa246d
20 changed files with 979 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
etc/proxy-server.conf-sample
View File

@ -69,7 +69,7 @@
# eventlet_debug = false
[pipeline:main]
pipeline = catch_errors healthcheck proxy-logging cache bulk slo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk slo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server
[app:proxy-server]
use = egg:swift#proxy
@ -509,3 +509,12 @@ use = egg:swift#slo
[filter:account-quotas]
use = egg:swift#account_quotas
[filter:gatekeeper]
use = egg:swift#gatekeeper
# You can override the default log routing for this filter here:
# set log_name = gatekeeper
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = false
# set log_address = /dev/log

1
setup.cfg
View File

@ -86,6 +86,7 @@ paste.filter_factory =
proxy_logging = swift.common.middleware.proxy_logging:filter_factory
slo = swift.common.middleware.slo:filter_factory
list_endpoints = swift.common.middleware.list_endpoints:filter_factory
gatekeeper = swift.common.middleware.gatekeeper:filter_factory
[build_sphinx]
all_files = 1

5
swift/account/server.py
View File

@ -37,6 +37,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, \
HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, \
HTTPPreconditionFailed, HTTPConflict, Request, \
HTTPInsufficientStorage, HTTPException
from swift.common.request_helpers import is_sys_or_user_meta
DATADIR = 'accounts'
@ -152,7 +153,7 @@ class AccountController(object):
metadata = {}
metadata.update((key, (value, timestamp))
for key, value in req.headers.iteritems()
if key.lower().startswith('x-account-meta-'))
if is_sys_or_user_meta('account', key))
if metadata:
broker.update_metadata(metadata)
if created:
@ -258,7 +259,7 @@ class AccountController(object):
metadata = {}
metadata.update((key, (value, timestamp))
for key, value in req.headers.iteritems()
if key.lower().startswith('x-account-meta-'))
if is_sys_or_user_meta('account', key))
if metadata:
broker.update_metadata(metadata)
return HTTPNoContent(request=req)

94
swift/common/middleware/gatekeeper.py Normal file
View File

@ -0,0 +1,94 @@
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
The ``gatekeeper`` middleware imposes restrictions on the headers that
may be included with requests and responses. Request headers are filtered
to remove headers that should never be generated by a client. Similarly,
response headers are filtered to remove private headers that should
never be passed to a client.
The ``gatekeeper`` middleware must always be present in the proxy server
wsgi pipeline. It should be configured close to the start of the pipeline
specified in ``/etc/swift/proxy-server.conf``, immediately after catch_errors
and before any other middleware. It is essential that it is configured ahead
of all middlewares using system metadata in order that they function
correctly.
If ``gatekeeper`` middleware is not configured in the pipeline then it will be
automatically inserted close to the start of the pipeline by the proxy server.
"""
from swift.common.swob import wsgify
from swift.common.utils import get_logger
from swift.common.request_helpers import remove_items, get_sys_meta_prefix
import re
"""
A list of python regular expressions that will be used to
match against inbound request headers. Matching headers will
be removed from the request.
"""
# Exclude headers starting with a sysmeta prefix.
# If adding to this list, note that these are regex patterns,
# so use a trailing $ to constrain to an exact header match
# rather than prefix match.
inbound_exclusions = [get_sys_meta_prefix('account'),
get_sys_meta_prefix('container'),
get_sys_meta_prefix('object')]
# 'x-object-sysmeta' is reserved in anticipation of future support
# for system metadata being applied to objects
"""
A list of python regular expressions that will be used to
match against outbound response headers. Matching headers will
be removed from the response.
"""
outbound_exclusions = inbound_exclusions
def make_exclusion_test(exclusions):
expr = '|'.join(exclusions)
test = re.compile(expr, re.IGNORECASE)
return test.match
class GatekeeperMiddleware(object):
def __init__(self, app, conf):
self.app = app
self.logger = get_logger(conf, log_route='gatekeeper')
self.inbound_condition = make_exclusion_test(inbound_exclusions)
self.outbound_condition = make_exclusion_test(outbound_exclusions)
@wsgify
def __call__(self, req):
removed = remove_items(req.headers, self.inbound_condition)
if removed:
self.logger.debug('removed request headers: %s' % removed)
resp = req.get_response(self.app)
removed = remove_items(resp.headers, self.outbound_condition)
if removed:
self.logger.debug('removed response headers: %s' % removed)
return resp
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def gatekeeper_filter(app):
return GatekeeperMiddleware(app, conf)
return gatekeeper_filter

106
swift/common/request_helpers.py
View File

@ -87,3 +87,109 @@ def split_and_validate_path(request, minsegs=1, maxsegs=None,
except ValueError as err:
raise HTTPBadRequest(body=str(err), request=request,
content_type='text/plain')
def is_user_meta(server_type, key):
"""
Tests if a header key starts with and is longer than the user
metadata prefix for given server type.
:param server_type: type of backend server i.e. [account|container|object]
:param key: header key
:returns: True if the key satisfies the test, False otherwise
"""
if len(key) <= 8 + len(server_type):
return False
return key.lower().startswith(get_user_meta_prefix(server_type))
def is_sys_meta(server_type, key):
"""
Tests if a header key starts with and is longer than the system
metadata prefix for given server type.
:param server_type: type of backend server i.e. [account|container|object]
:param key: header key
:returns: True if the key satisfies the test, False otherwise
"""
if len(key) <= 11 + len(server_type):
return False
return key.lower().startswith(get_sys_meta_prefix(server_type))
def is_sys_or_user_meta(server_type, key):
"""
Tests if a header key starts with and is longer than the user or system
metadata prefix for given server type.
:param server_type: type of backend server i.e. [account|container|object]
:param key: header key
:returns: True if the key satisfies the test, False otherwise
"""
return is_user_meta(server_type, key) or is_sys_meta(server_type, key)
def strip_user_meta_prefix(server_type, key):
"""
Removes the user metadata prefix for a given server type from the start
of a header key.
:param server_type: type of backend server i.e. [account|container|object]
:param key: header key
:returns: stripped header key
"""
return key[len(get_user_meta_prefix(server_type)):]
def strip_sys_meta_prefix(server_type, key):
"""
Removes the system metadata prefix for a given server type from the start
of a header key.
:param server_type: type of backend server i.e. [account|container|object]
:param key: header key
:returns: stripped header key
"""
return key[len(get_sys_meta_prefix(server_type)):]
def get_user_meta_prefix(server_type):
"""
Returns the prefix for user metadata headers for given server type.
This prefix defines the namespace for headers that will be persisted
by backend servers.
:param server_type: type of backend server i.e. [account|container|object]
:returns: prefix string for server type's user metadata headers
"""
return 'x-%s-%s-' % (server_type.lower(), 'meta')
def get_sys_meta_prefix(server_type):
"""
Returns the prefix for system metadata headers for given server type.
This prefix defines the namespace for headers that will be persisted
by backend servers.
:param server_type: type of backend server i.e. [account|container|object]
:returns: prefix string for server type's system metadata headers
"""
return 'x-%s-%s-' % (server_type.lower(), 'sysmeta')
def remove_items(headers, condition):
"""
Removes items from a dict whose keys satisfy
the given condition.
:param headers: a dict of headers
:param condition: a function that will be passed the header key as a
single argument and should return True if the header is to be removed.
:returns: a dict, possibly empty, of headers that have been removed
"""
removed = {}
keys = filter(condition, headers)
removed.update((key, headers.pop(key)) for key in keys)
return removed

15
swift/common/wsgi.py
View File

@ -213,6 +213,21 @@ class PipelineWrapper(object):
except ValueError:
return False
def startswith(self, entry_point_name):
"""
Tests if the pipeline starts with the given entry point name.
:param entry_point_name: entry point of middleware or app (Swift only)
:returns: True if entry_point_name is first in pipeline, False
otherwise
"""
try:
first_ctx = self.context.filter_contexts[0]
except IndexError:
first_ctx = self.context.app_context
return first_ctx.entry_point_name == entry_point_name
def _format_for_display(self, ctx):
if ctx.entry_point_name:
return ctx.entry_point_name

10
swift/container/server.py
View File

@ -26,7 +26,7 @@ import swift.common.db
from swift.container.backend import ContainerBroker
from swift.common.db import DatabaseAlreadyExists
from swift.common.request_helpers import get_param, get_listing_content_type, \
split_and_validate_path
split_and_validate_path, is_sys_or_user_meta
from swift.common.utils import get_logger, hash_path, public, \
normalize_timestamp, storage_directory, validate_sync_to, \
config_true_value, json, timing_stats, replication, \
@ -266,7 +266,7 @@ class ContainerController(object):
(key, (value, timestamp))
for key, value in req.headers.iteritems()
if key.lower() in self.save_headers or
key.lower().startswith('x-container-meta-'))
is_sys_or_user_meta('container', key))
if metadata:
if 'X-Container-Sync-To' in metadata:
if 'X-Container-Sync-To' not in broker.metadata or \
@ -307,7 +307,7 @@ class ContainerController(object):
(key, value)
for key, (value, timestamp) in broker.metadata.iteritems()
if value != '' and (key.lower() in self.save_headers or
key.lower().startswith('x-container-meta-')))
is_sys_or_user_meta('container', key)))
headers['Content-Type'] = out_content_type
return HTTPNoContent(request=req, headers=headers, charset='utf-8')
@ -374,7 +374,7 @@ class ContainerController(object):
}
for key, (value, timestamp) in broker.metadata.iteritems():
if value and (key.lower() in self.save_headers or
key.lower().startswith('x-container-meta-')):
is_sys_or_user_meta('container', key)):
resp_headers[key] = value
ret = Response(request=req, headers=resp_headers,
content_type=out_content_type, charset='utf-8')
@ -452,7 +452,7 @@ class ContainerController(object):
metadata.update(
(key, (value, timestamp)) for key, value in req.headers.iteritems()
if key.lower() in self.save_headers or
key.lower().startswith('x-container-meta-'))
is_sys_or_user_meta('container', key))
if metadata:
if 'X-Container-Sync-To' in metadata:
if 'X-Container-Sync-To' not in broker.metadata or \

11
swift/obj/server.py
View File

@ -38,7 +38,7 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \
DiskFileDeviceUnavailable, DiskFileExpired
from swift.obj import ssync_receiver
from swift.common.http import is_success
from swift.common.request_helpers import split_and_validate_path
from swift.common.request_helpers import split_and_validate_path, is_user_meta
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
HTTPInternalServerError, HTTPNoContent, HTTPNotFound, HTTPNotModified, \
HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
@ -338,7 +338,7 @@ class ObjectController(object):
return HTTPConflict(request=request)
metadata = {'X-Timestamp': request.headers['x-timestamp']}
metadata.update(val for val in request.headers.iteritems()
if val[0].startswith('X-Object-Meta-'))
if is_user_meta('object', val[0]))
for header_key in self.allowed_headers:
if header_key in request.headers:
header_caps = header_key.title()
@ -422,8 +422,7 @@ class ObjectController(object):
'Content-Length': str(upload_size),
}
metadata.update(val for val in request.headers.iteritems()
if val[0].lower().startswith('x-object-meta-')
and len(val[0]) > 14)
if is_user_meta('object', val[0]))
for header_key in (
request.headers.get('X-Backend-Replication-Headers') or
self.allowed_headers):
@ -504,7 +503,7 @@ class ObjectController(object):
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
if is_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
@ -545,7 +544,7 @@ class ObjectController(object):
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
if is_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']

55
swift/proxy/controllers/base.py
View File

@ -48,6 +48,8 @@ from swift.common.http import is_informational, is_success, is_redirection, \
HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED
from swift.common.swob import Request, Response, HeaderKeyDict, Range, \
HTTPException, HTTPRequestedRangeNotSatisfiable
from swift.common.request_helpers import strip_sys_meta_prefix, \
strip_user_meta_prefix, is_user_meta, is_sys_meta, is_sys_or_user_meta
def update_headers(response, headers):
@ -106,11 +108,32 @@ def get_container_memcache_key(account, container):
return cache_key
def _prep_headers_to_info(headers, server_type):
"""
Helper method that iterates once over a dict of headers,
converting all keys to lower case and separating
into subsets containing user metadata, system metadata
and other headers.
"""
meta = {}
sysmeta = {}
other = {}
for key, val in dict(headers).iteritems():
lkey = key.lower()
if is_user_meta(server_type, lkey):
meta[strip_user_meta_prefix(server_type, lkey)] = val
elif is_sys_meta(server_type, lkey):
sysmeta[strip_sys_meta_prefix(server_type, lkey)] = val
else:
other[lkey] = val
return other, meta, sysmeta
def headers_to_account_info(headers, status_int=HTTP_OK):
"""
Construct a cacheable dict of account info based on response headers.
"""
headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
headers, meta, sysmeta = _prep_headers_to_info(headers, 'account')
return {
'status': status_int,
# 'container_count' anomaly:
@ -120,9 +143,8 @@ def headers_to_account_info(headers, status_int=HTTP_OK):
'container_count': headers.get('x-account-container-count'),
'total_object_count': headers.get('x-account-object-count'),
'bytes': headers.get('x-account-bytes-used'),
'meta': dict((key[15:], value)
for key, value in headers.iteritems()
if key.startswith('x-account-meta-'))
'meta': meta,
'sysmeta': sysmeta
}
@ -130,7 +152,7 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
"""
Construct a cacheable dict of container info based on response headers.
"""
headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
headers, meta, sysmeta = _prep_headers_to_info(headers, 'container')
return {
'status': status_int,
'read_acl': headers.get('x-container-read'),
@ -140,16 +162,12 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
'bytes': headers.get('x-container-bytes-used'),
'versions': headers.get('x-versions-location'),
'cors': {
'allow_origin': headers.get(
'x-container-meta-access-control-allow-origin'),
'expose_headers': headers.get(
'x-container-meta-access-control-expose-headers'),
'max_age': headers.get(
'x-container-meta-access-control-max-age')
'allow_origin': meta.get('access-control-allow-origin'),
'expose_headers': meta.get('access-control-expose-headers'),
'max_age': meta.get('access-control-max-age')
},
'meta': dict((key[17:], value)
for key, value in headers.iteritems()
if key.startswith('x-container-meta-'))
'meta': meta,
'sysmeta': sysmeta
}
@ -157,14 +175,12 @@ def headers_to_object_info(headers, status_int=HTTP_OK):
"""
Construct a cacheable dict of object info based on response headers.
"""
headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
headers, meta, sysmeta = _prep_headers_to_info(headers, 'object')
info = {'status': status_int,
'length': headers.get('content-length'),
'type': headers.get('content-type'),
'etag': headers.get('etag'),
'meta': dict((key[14:], value)
for key, value in headers.iteritems()
if key.startswith('x-object-meta-'))
'meta': meta
}
return info
@ -854,11 +870,10 @@ class Controller(object):
if k.lower().startswith(x_remove) or
k.lower() in self._x_remove_headers())
x_meta = 'x-%s-meta-' % st
dst_headers.update((k.lower(), v)
for k, v in src_headers.iteritems()
if k.lower() in self.pass_through_headers or
k.lower().startswith(x_meta))
is_sys_or_user_meta(st, k))
def generate_request_headers(self, orig_req=None, additional=None,
transfer=False):

3
swift/proxy/controllers/obj.py
View File

@ -59,6 +59,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \
HTTPServerError, HTTPServiceUnavailable, Request, Response, \
HTTPClientDisconnect, HTTPNotImplemented, HTTPException
from swift.common.request_helpers import is_user_meta
def segment_listing_iter(listing):
@ -78,7 +79,7 @@ def copy_headers_into(from_r, to_r):
"""
pass_headers = ['x-delete-at']
for k, v in from_r.headers.items():
if k.lower().startswith('x-object-meta-') or k.lower() in pass_headers:
if is_user_meta('object', k) or k.lower() in pass_headers:
to_r.headers[k] = v

17
swift/proxy/server.py
View File

@ -51,7 +51,17 @@ from swift.common.swob import HTTPBadRequest, HTTPForbidden, \
# example, 'after: ["catch_errors", "bulk"]' would install this middleware
# after catch_errors and bulk if both were present, but if bulk were absent,
# would just install it after catch_errors.
required_filters = [{'name': 'catch_errors'}]
#
# "after_fn" (optional) a function that takes a PipelineWrapper object as its
# single argument and returns a list of middlewares that this middleware should
# come after. This list overrides any defined by the "after" field.
required_filters = [
{'name': 'catch_errors'},
{'name': 'gatekeeper',
'after_fn': lambda pipe: (['catch_errors']
if pipe.startswith("catch_errors")
else [])}
]
class Application(object):
@ -505,7 +515,10 @@ class Application(object):
for filter_spec in reversed(required_filters):
filter_name = filter_spec['name']
if filter_name not in pipe:
afters = filter_spec.get('after', [])
if 'after_fn' in filter_spec:
afters = filter_spec['after_fn'](pipe)
else:
afters = filter_spec.get('after', [])
insert_at = 0
for after in afters:
try:

115
test/unit/account/test_server.py
View File

@ -26,6 +26,7 @@ import xml.dom.minidom
from swift.common.swob import Request
from swift.account.server import AccountController, ACCOUNT_LISTING_LIMIT
from swift.common.utils import normalize_timestamp, replication, public
from swift.common.request_helpers import get_sys_meta_prefix
class TestAccountController(unittest.TestCase):
@ -371,6 +372,67 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(resp.status_int, 204)
self.assert_('x-account-meta-test' not in resp.headers)
def test_PUT_GET_sys_metadata(self):
prefix = get_sys_meta_prefix('account')
hdr = '%stest' % prefix
hdr2 = '%stest2' % prefix
# Set metadata header
req = Request.blank(
'/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(1),
hdr.title(): 'Value'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
self.assertEqual(resp.headers.get(hdr), 'Value')
# Set another metadata header, ensuring old one doesn't disappear
req = Request.blank(
'/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': normalize_timestamp(1),
hdr2.title(): 'Value2'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
self.assertEqual(resp.headers.get(hdr), 'Value')
self.assertEqual(resp.headers.get(hdr2), 'Value2')
# Update metadata header
req = Request.blank(
'/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(3),
hdr.title(): 'New Value'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 202)
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
self.assertEqual(resp.headers.get(hdr), 'New Value')
# Send old update to metadata header
req = Request.blank(
'/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(2),
hdr.title(): 'Old Value'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 202)
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
self.assertEqual(resp.headers.get(hdr), 'New Value')
# Remove metadata header (by setting it to empty)
req = Request.blank(
'/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(4),
hdr.title(): ''})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 202)
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
self.assert_(hdr not in resp.headers)
def test_PUT_invalid_partition(self):
req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1'})
@ -435,6 +497,59 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(resp.status_int, 204)
self.assert_('x-account-meta-test' not in resp.headers)
def test_POST_HEAD_sys_metadata(self):
prefix = get_sys_meta_prefix('account')
hdr = '%stest' % prefix
req = Request.blank(
'/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(1)})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
# Set metadata header
req = Request.blank(
'/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': normalize_timestamp(1),
hdr.title(): 'Value'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
self.assertEqual(resp.headers.get(hdr), 'Value')
# Update metadata header
req = Request.blank(
'/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': normalize_timestamp(3),
hdr.title(): 'New Value'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
self.assertEqual(resp.headers.get(hdr), 'New Value')
# Send old update to metadata header
req = Request.blank(
'/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': normalize_timestamp(2),
hdr.title(): 'Old Value'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
self.assertEqual(resp.headers.get(hdr), 'New Value')
# Remove metadata header (by setting it to empty)
req = Request.blank(
'/sda1/p/a', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': normalize_timestamp(4),
hdr.title(): ''})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204)
self.assert_(hdr not in resp.headers)
def test_POST_invalid_partition(self):
req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'POST',
'HTTP_X_TIMESTAMP': '1'})

115
test/unit/common/middleware/test_gatekeeper.py Normal file
View File

@ -0,0 +1,115 @@
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from swift.common.swob import Request, Response
from swift.common.middleware import gatekeeper
class FakeApp(object):
def __init__(self, headers={}):
self.headers = headers
self.req = None
def __call__(self, env, start_response):
self.req = Request(env)
return Response(request=self.req, body='FAKE APP',
headers=self.headers)(env, start_response)
class TestGatekeeper(unittest.TestCase):
methods = ['PUT', 'POST', 'GET', 'DELETE', 'HEAD', 'COPY', 'OPTIONS']
allowed_headers = {'xx-account-sysmeta-foo': 'value',
'xx-container-sysmeta-foo': 'value',
'xx-object-sysmeta-foo': 'value',
'x-account-meta-foo': 'value',
'x-container-meta-foo': 'value',
'x-object-meta-foo': 'value',
'x-timestamp-foo': 'value'}
sysmeta_headers = {'x-account-sysmeta-': 'value',
'x-container-sysmeta-': 'value',
'x-object-sysmeta-': 'value',
'x-account-sysmeta-foo': 'value',
'x-container-sysmeta-foo': 'value',
'x-object-sysmeta-foo': 'value',
'X-Account-Sysmeta-BAR': 'value',
'X-Container-Sysmeta-BAR': 'value',
'X-Object-Sysmeta-BAR': 'value'}
forbidden_headers_out = dict(sysmeta_headers)
forbidden_headers_in = dict(sysmeta_headers)
def _assertHeadersEqual(self, expected, actual):
for key in expected:
self.assertTrue(key.lower() in actual,
'%s missing from %s' % (key, actual))
def _assertHeadersAbsent(self, unexpected, actual):
for key in unexpected:
self.assertTrue(key.lower() not in actual,
'%s is in %s' % (key, actual))
def get_app(self, app, global_conf, **local_conf):
factory = gatekeeper.filter_factory(global_conf, **local_conf)
return factory(app)
def test_ok_header(self):
req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': 'PUT'},
headers=self.allowed_headers)
fake_app = FakeApp()
app = self.get_app(fake_app, {})
resp = req.get_response(app)
self.assertEquals('200 OK', resp.status)
self.assertEquals(resp.body, 'FAKE APP')
self._assertHeadersEqual(self.allowed_headers, fake_app.req.headers)
def _test_reserved_header_removed_inbound(self, method):
headers = dict(self.forbidden_headers_in)
headers.update(self.allowed_headers)
req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': method},
headers=headers)
fake_app = FakeApp()
app = self.get_app(fake_app, {})
resp = req.get_response(app)
self.assertEquals('200 OK', resp.status)
self._assertHeadersEqual(self.allowed_headers, fake_app.req.headers)
self._assertHeadersAbsent(self.forbidden_headers_in,
fake_app.req.headers)
def test_reserved_header_removed_inbound(self):
for method in self.methods:
self._test_reserved_header_removed_inbound(method)
def _test_reserved_header_removed_outbound(self, method):
headers = dict(self.forbidden_headers_out)
headers.update(self.allowed_headers)
req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': method})
fake_app = FakeApp(headers=headers)
app = self.get_app(fake_app, {})
resp = req.get_response(app)
self.assertEquals('200 OK', resp.status)
self._assertHeadersEqual(self.allowed_headers, resp.headers)
self._assertHeadersAbsent(self.forbidden_headers_out, resp.headers)
def test_reserved_header_removed_outbound(self):
for method in self.methods:
self._test_reserved_header_removed_outbound(method)
if __name__ == '__main__':
unittest.main()

70
test/unit/common/test_request_helpers.py Normal file
View File

@ -0,0 +1,70 @@
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for swift.common.request_helpers"""
import unittest
from swift.common.request_helpers import is_sys_meta, is_user_meta, \
is_sys_or_user_meta, strip_sys_meta_prefix, strip_user_meta_prefix, \
remove_items
server_types = ['account', 'container', 'object']
class TestRequestHelpers(unittest.TestCase):
def test_is_user_meta(self):
m_type = 'meta'
for st in server_types:
self.assertTrue(is_user_meta(st, 'x-%s-%s-foo' % (st, m_type)))
self.assertFalse(is_user_meta(st, 'x-%s-%s-' % (st, m_type)))
self.assertFalse(is_user_meta(st, 'x-%s-%sfoo' % (st, m_type)))
def test_is_sys_meta(self):
m_type = 'sysmeta'
for st in server_types:
self.assertTrue(is_sys_meta(st, 'x-%s-%s-foo' % (st, m_type)))
self.assertFalse(is_sys_meta(st, 'x-%s-%s-' % (st, m_type)))
self.assertFalse(is_sys_meta(st, 'x-%s-%sfoo' % (st, m_type)))
def test_is_sys_or_user_meta(self):
m_types = ['sysmeta', 'meta']
for mt in m_types:
for st in server_types:
self.assertTrue(is_sys_or_user_meta(st, 'x-%s-%s-foo'
% (st, mt)))
self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%s-'
% (st, mt)))
self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%sfoo'
% (st, mt)))
def test_strip_sys_meta_prefix(self):
mt = 'sysmeta'
for st in server_types:
self.assertEquals(strip_sys_meta_prefix(st, 'x-%s-%s-a'
% (st, mt)), 'a')
def test_strip_user_meta_prefix(self):
mt = 'meta'
for st in server_types:
self.assertEquals(strip_user_meta_prefix(st, 'x-%s-%s-a'
% (st, mt)), 'a')
def test_remove_items(self):
src = {'a': 'b',
'c': 'd'}
test = lambda x: x == 'a'
rem = remove_items(src, test)
self.assertEquals(src, {'c': 'd'})
self.assertEquals(rem, {'a': 'b'})

105
test/unit/common/test_wsgi.py
View File

@ -35,6 +35,7 @@ from eventlet import listen
import mock
import swift.common.middleware.catch_errors
import swift.common.middleware.gatekeeper
import swift.proxy.server
from swift.common.swob import Request
@ -143,6 +144,9 @@ class TestWSGI(unittest.TestCase):
# verify pipeline is catch_errors -> proxy-server
expected = swift.common.middleware.catch_errors.CatchErrorMiddleware
self.assert_(isinstance(app, expected))
app = app.app
expected = swift.common.middleware.gatekeeper.GatekeeperMiddleware
self.assert_(isinstance(app, expected))
self.assert_(isinstance(app.app, swift.proxy.server.Application))
# config settings applied to app instance
self.assertEquals(0.2, app.app.conn_timeout)
@ -706,6 +710,31 @@ class TestPipelineWrapper(unittest.TestCase):
# filters in the pipeline.
return [c.entry_point_name for c in self.pipe.context.filter_contexts]
def test_startswith(self):
self.assertTrue(self.pipe.startswith("healthcheck"))
self.assertFalse(self.pipe.startswith("tempurl"))
def test_startswith_no_filters(self):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = proxy-server
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
"""
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
ctx = wsgi.loadcontext(loadwsgi.APP, conf_file, global_conf={})
pipe = wsgi.PipelineWrapper(ctx)
self.assertTrue(pipe.startswith('proxy'))
def test_insert_filter(self):
original_modules = ['healthcheck', 'catch_errors', None]
self.assertEqual(self._entry_point_names(), original_modules)
@ -789,7 +818,7 @@ class TestPipelineModification(unittest.TestCase):
swift_dir = TEMPDIR
[pipeline:main]
pipeline = catch_errors proxy-server
pipeline = catch_errors gatekeeper proxy-server
[app:proxy-server]
use = egg:swift#proxy
@ -797,6 +826,9 @@ class TestPipelineModification(unittest.TestCase):
[filter:catch_errors]
use = egg:swift#catch_errors
[filter:gatekeeper]
use = egg:swift#gatekeeper
"""
contents = dedent(config)
@ -809,6 +841,7 @@ class TestPipelineModification(unittest.TestCase):
self.assertEqual(self.pipeline_modules(app),
['swift.common.middleware.catch_errors',
'swift.common.middleware.gatekeeper',
'swift.proxy.server'])
def test_proxy_modify_wsgi_pipeline(self):
@ -835,8 +868,11 @@ class TestPipelineModification(unittest.TestCase):
_fake_rings(t)
app = wsgi.loadapp(conf_file, global_conf={})
self.assertEqual(self.pipeline_modules(app)[0],
'swift.common.middleware.catch_errors')
self.assertEqual(self.pipeline_modules(app),
['swift.common.middleware.catch_errors',
'swift.common.middleware.gatekeeper',
'swift.common.middleware.healthcheck',
'swift.proxy.server'])
def test_proxy_modify_wsgi_pipeline_ordering(self):
config = """
@ -892,6 +928,69 @@ class TestPipelineModification(unittest.TestCase):
'swift.common.middleware.tempurl',
'swift.proxy.server'])
def _proxy_modify_wsgi_pipeline(self, pipe):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = %s
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:catch_errors]
use = egg:swift#catch_errors
[filter:gatekeeper]
use = egg:swift#gatekeeper
"""
config = config % (pipe,)
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
app = wsgi.loadapp(conf_file, global_conf={})
return app
def test_gatekeeper_insertion_catch_errors_configured_at_start(self):
# catch_errors is configured at start, gatekeeper is not configured,
# so gatekeeper should be inserted just after catch_errors
pipe = 'catch_errors healthcheck proxy-server'
app = self._proxy_modify_wsgi_pipeline(pipe)
self.assertEqual(self.pipeline_modules(app), [
'swift.common.middleware.catch_errors',
'swift.common.middleware.gatekeeper',
'swift.common.middleware.healthcheck',
'swift.proxy.server'])
def test_gatekeeper_insertion_catch_errors_configured_not_at_start(self):
# catch_errors is configured, gatekeeper is not configured, so
# gatekeeper should be inserted at start of pipeline
pipe = 'healthcheck catch_errors proxy-server'
app = self._proxy_modify_wsgi_pipeline(pipe)
self.assertEqual(self.pipeline_modules(app), [
'swift.common.middleware.gatekeeper',
'swift.common.middleware.healthcheck',
'swift.common.middleware.catch_errors',
'swift.proxy.server'])
def test_catch_errors_gatekeeper_configured_not_at_start(self):
# catch_errors is configured, gatekeeper is configured, so
# no change should be made to pipeline
pipe = 'healthcheck catch_errors gatekeeper proxy-server'
app = self._proxy_modify_wsgi_pipeline(pipe)
self.assertEqual(self.pipeline_modules(app), [
'swift.common.middleware.healthcheck',
'swift.common.middleware.catch_errors',
'swift.common.middleware.gatekeeper',
'swift.proxy.server'])
if __name__ == '__main__':
unittest.main()

109
test/unit/container/test_server.py
View File

@ -31,6 +31,7 @@ import swift.container
from swift.container import server as container_server
from swift.common.utils import normalize_timestamp, mkdirs, public, replication
from test.unit import fake_http_connect
from swift.common.request_helpers import get_sys_meta_prefix
@contextmanager
@ -292,6 +293,64 @@ class TestContainerController(unittest.TestCase):
self.assertEquals(resp.status_int, 204)
self.assert_('x-container-meta-test' not in resp.headers)
def test_PUT_GET_sys_metadata(self):
prefix = get_sys_meta_prefix('container')
key = '%sTest' % prefix
key2 = '%sTest2' % prefix
# Set metadata header
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(1),
key: 'Value'})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 201)
req = Request.blank('/sda1/p/a/c')
resp = self.controller.GET(req)
self.assertEquals(resp.status_int, 204)
self.assertEquals(resp.headers.get(key.lower()), 'Value')
# Set another metadata header, ensuring old one doesn't disappear
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': normalize_timestamp(1),
key2: 'Value2'})
resp = self.controller.POST(req)
self.assertEquals(resp.status_int, 204)
req = Request.blank('/sda1/p/a/c')
resp = self.controller.GET(req)
self.assertEquals(resp.status_int, 204)
self.assertEquals(resp.headers.get(key.lower()), 'Value')
self.assertEquals(resp.headers.get(key2.lower()), 'Value2')
# Update metadata header
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(3),
key: 'New Value'})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 202)
req = Request.blank('/sda1/p/a/c')
resp = self.controller.GET(req)
self.assertEquals(resp.status_int, 204)
self.assertEquals(resp.headers.get(key.lower()),
'New Value')
# Send old update to metadata header
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(2),
key: 'Old Value'})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 202)
req = Request.blank('/sda1/p/a/c')
resp = self.controller.GET(req)
self.assertEquals(resp.status_int, 204)
self.assertEquals(resp.headers.get(key.lower()),
'New Value')
# Remove metadata header (by setting it to empty)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(4),
key: ''})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 202)
req = Request.blank('/sda1/p/a/c')
resp = self.controller.GET(req)
self.assertEquals(resp.status_int, 204)
self.assert_(key.lower() not in resp.headers)
def test_PUT_invalid_partition(self):
req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1'})
@ -369,6 +428,56 @@ class TestContainerController(unittest.TestCase):
self.assertEquals(resp.status_int, 204)
self.assert_('x-container-meta-test' not in resp.headers)
def test_POST_HEAD_sys_metadata(self):
prefix = get_sys_meta_prefix('container')
key = '%sTest' % prefix
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(1)})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 201)
# Set metadata header
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': normalize_timestamp(1),
key: 'Value'})
resp = self.controller.POST(req)
self.assertEquals(resp.status_int, 204)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
resp = self.controller.HEAD(req)
self.assertEquals(resp.status_int, 204)
self.assertEquals(resp.headers.get(key.lower()), 'Value')
# Update metadata header
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': normalize_timestamp(3),
key: 'New Value'})
resp = self.controller.POST(req)
self.assertEquals(resp.status_int, 204)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
resp = self.controller.HEAD(req)
self.assertEquals(resp.status_int, 204)
self.assertEquals(resp.headers.get(key.lower()),
'New Value')
# Send old update to metadata header
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': normalize_timestamp(2),
key: 'Old Value'})
resp = self.controller.POST(req)
self.assertEquals(resp.status_int, 204)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
resp = self.controller.HEAD(req)
self.assertEquals(resp.status_int, 204)
self.assertEquals(resp.headers.get(key.lower()),
'New Value')
# Remove metadata header (by setting it to empty)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': normalize_timestamp(4),
key: ''})
resp = self.controller.POST(req)
self.assertEquals(resp.status_int, 204)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
resp = self.controller.HEAD(req)
self.assertEquals(resp.status_int, 204)
self.assert_(key.lower() not in resp.headers)
def test_POST_invalid_partition(self):
req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'POST',
'HTTP_X_TIMESTAMP': '1'})

57
test/unit/proxy/controllers/test_account.py
View File

@ -21,6 +21,7 @@ from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_account_info
from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH as MAX_ANAME_LEN
from test.unit import fake_http_connect, FakeRing, FakeMemcache
from swift.common.request_helpers import get_sys_meta_prefix
class TestAccountController(unittest.TestCase):
@ -95,6 +96,62 @@ class TestAccountController(unittest.TestCase):
resp = controller.POST(req)
self.assertEquals(400, resp.status_int)
def _make_callback_func(self, context):
def callback(ipaddr, port, device, partition, method, path,
headers=None, query_string=None, ssl=False):
context['method'] = method
context['path'] = path
context['headers'] = headers or {}
return callback
def test_sys_meta_headers_PUT(self):
# check that headers in sys meta namespace make it through
# the proxy controller
sys_meta_key = '%stest' % get_sys_meta_prefix('account')
sys_meta_key = sys_meta_key.title()
user_meta_key = 'X-Account-Meta-Test'
# allow PUTs to account...
self.app.allow_account_management = True
controller = proxy_server.AccountController(self.app, 'a')
context = {}
callback = self._make_callback_func(context)
hdrs_in = {sys_meta_key: 'foo',
user_meta_key: 'bar',
'x-timestamp': '1.0'}
req = Request.blank('/v1/a', headers=hdrs_in)
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, give_connect=callback)):
controller.PUT(req)
self.assertEqual(context['method'], 'PUT')
self.assertTrue(sys_meta_key in context['headers'])
self.assertEqual(context['headers'][sys_meta_key], 'foo')
self.assertTrue(user_meta_key in context['headers'])
self.assertEqual(context['headers'][user_meta_key], 'bar')
self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
def test_sys_meta_headers_POST(self):
# check that headers in sys meta namespace make it through
# the proxy controller
sys_meta_key = '%stest' % get_sys_meta_prefix('account')
sys_meta_key = sys_meta_key.title()
user_meta_key = 'X-Account-Meta-Test'
controller = proxy_server.AccountController(self.app, 'a')
context = {}
callback = self._make_callback_func(context)
hdrs_in = {sys_meta_key: 'foo',
user_meta_key: 'bar',
'x-timestamp': '1.0'}
req = Request.blank('/v1/a', headers=hdrs_in)
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, give_connect=callback)):
controller.POST(req)
self.assertEqual(context['method'], 'POST')
self.assertTrue(sys_meta_key in context['headers'])
self.assertEqual(context['headers'][sys_meta_key], 'foo')
self.assertTrue(user_meta_key in context['headers'])
self.assertEqual(context['headers'][user_meta_key], 'bar')
self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
if __name__ == '__main__':
unittest.main()

61
test/unit/proxy/controllers/test_base.py
View File

@ -20,10 +20,11 @@ from swift.proxy.controllers.base import headers_to_container_info, \
get_container_memcache_key, get_account_info, get_account_memcache_key, \
get_object_env_key, _get_cache_key, get_info, get_object_info, \
Controller, GetOrHeadHandler
from swift.common.swob import Request, HTTPException
from swift.common.swob import Request, HTTPException, HeaderKeyDict
from swift.common.utils import split_path
from test.unit import fake_http_connect, FakeRing, FakeMemcache
from swift.proxy import server as proxy_server
from swift.common.request_helpers import get_sys_meta_prefix
FakeResponse_status_int = 201
@ -365,6 +366,15 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(resp['meta']['whatevs'], 14)
self.assertEquals(resp['meta']['somethingelse'], 0)
def test_headers_to_container_info_sys_meta(self):
prefix = get_sys_meta_prefix('container')
headers = {'%sWhatevs' % prefix: 14,
'%ssomethingelse' % prefix: 0}
resp = headers_to_container_info(headers.items(), 200)
self.assertEquals(len(resp['sysmeta']), 2)
self.assertEquals(resp['sysmeta']['whatevs'], 14)
self.assertEquals(resp['sysmeta']['somethingelse'], 0)
def test_headers_to_container_info_values(self):
headers = {
'x-container-read': 'readvalue',
@ -396,6 +406,15 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(resp['meta']['whatevs'], 14)
self.assertEquals(resp['meta']['somethingelse'], 0)
def test_headers_to_account_info_sys_meta(self):
prefix = get_sys_meta_prefix('account')
headers = {'%sWhatevs' % prefix: 14,
'%ssomethingelse' % prefix: 0}
resp = headers_to_account_info(headers.items(), 200)
self.assertEquals(len(resp['sysmeta']), 2)
self.assertEquals(resp['sysmeta']['whatevs'], 14)
self.assertEquals(resp['sysmeta']['somethingelse'], 0)
def test_headers_to_account_info_values(self):
headers = {
'x-account-object-count': '10',
@ -473,3 +492,43 @@ class TestFuncs(unittest.TestCase):
{'Range': 'bytes=-100'})
handler.fast_forward(20)
self.assertEquals(handler.backend_headers['Range'], 'bytes=-80')
def test_transfer_headers_with_sysmeta(self):
base = Controller(self.app)
good_hdrs = {'x-base-sysmeta-foo': 'ok',
'X-Base-sysmeta-Bar': 'also ok'}
bad_hdrs = {'x-base-sysmeta-': 'too short'}
hdrs = dict(good_hdrs)
hdrs.update(bad_hdrs)
dst_hdrs = HeaderKeyDict()
base.transfer_headers(hdrs, dst_hdrs)
self.assertEqual(HeaderKeyDict(good_hdrs), dst_hdrs)
def test_generate_request_headers(self):
base = Controller(self.app)
src_headers = {'x-remove-base-meta-owner': 'x',
'x-base-meta-size': '151M',
'new-owner': 'Kun'}
req = Request.blank('/v1/a/c/o', headers=src_headers)
dst_headers = base.generate_request_headers(req, transfer=True)
expected_headers = {'x-base-meta-owner': '',
'x-base-meta-size': '151M'}
for k, v in expected_headers.iteritems():
self.assertTrue(k in dst_headers)
self.assertEqual(v, dst_headers[k])
self.assertFalse('new-owner' in dst_headers)
def test_generate_request_headers_with_sysmeta(self):
base = Controller(self.app)
good_hdrs = {'x-base-sysmeta-foo': 'ok',
'X-Base-sysmeta-Bar': 'also ok'}
bad_hdrs = {'x-base-sysmeta-': 'too short'}
hdrs = dict(good_hdrs)
hdrs.update(bad_hdrs)
req = Request.blank('/v1/a/c/o', headers=hdrs)
dst_headers = base.generate_request_headers(req, transfer=True)
for k, v in good_hdrs.iteritems():
self.assertTrue(k.lower() in dst_headers)
self.assertEqual(v, dst_headers[k.lower()])
for k, v in bad_hdrs.iteritems():
self.assertFalse(k.lower() in dst_headers)

56
test/unit/proxy/controllers/test_container.py
View File

@ -20,6 +20,7 @@ from swift.common.swob import Request
from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_container_info
from test.unit import fake_http_connect, FakeRing, FakeMemcache
from swift.common.request_helpers import get_sys_meta_prefix
class TestContainerController(unittest.TestCase):
@ -62,6 +63,61 @@ class TestContainerController(unittest.TestCase):
for key in owner_headers:
self.assertTrue(key in resp.headers)
def _make_callback_func(self, context):
def callback(ipaddr, port, device, partition, method, path,
headers=None, query_string=None, ssl=False):
context['method'] = method
context['path'] = path
context['headers'] = headers or {}
return callback
def test_sys_meta_headers_PUT(self):
# check that headers in sys meta namespace make it through
# the container controller
sys_meta_key = '%stest' % get_sys_meta_prefix('container')
sys_meta_key = sys_meta_key.title()
user_meta_key = 'X-Container-Meta-Test'
controller = proxy_server.ContainerController(self.app, 'a', 'c')
context = {}
callback = self._make_callback_func(context)
hdrs_in = {sys_meta_key: 'foo',
user_meta_key: 'bar',
'x-timestamp': '1.0'}
req = Request.blank('/v1/a/c', headers=hdrs_in)
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, give_connect=callback)):
controller.PUT(req)
self.assertEqual(context['method'], 'PUT')
self.assertTrue(sys_meta_key in context['headers'])
self.assertEqual(context['headers'][sys_meta_key], 'foo')
self.assertTrue(user_meta_key in context['headers'])
self.assertEqual(context['headers'][user_meta_key], 'bar')
self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
def test_sys_meta_headers_POST(self):
# check that headers in sys meta namespace make it through
# the container controller
sys_meta_key = '%stest' % get_sys_meta_prefix('container')
sys_meta_key = sys_meta_key.title()
user_meta_key = 'X-Container-Meta-Test'
controller = proxy_server.ContainerController(self.app, 'a', 'c')
context = {}
callback = self._make_callback_func(context)
hdrs_in = {sys_meta_key: 'foo',
user_meta_key: 'bar',
'x-timestamp': '1.0'}
req = Request.blank('/v1/a/c', headers=hdrs_in)
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, give_connect=callback)):
controller.POST(req)
self.assertEqual(context['method'], 'POST')
self.assertTrue(sys_meta_key in context['headers'])
self.assertEqual(context['headers'][sys_meta_key], 'foo')
self.assertTrue(user_meta_key in context['headers'])
self.assertEqual(context['headers'][user_meta_key], 'bar')
self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
if __name__ == '__main__':
unittest.main()

6
test/unit/proxy/test_server.py
View File

@ -340,7 +340,8 @@ class TestController(unittest.TestCase):
'container_count': '12345',
'total_object_count': None,
'bytes': None,
'meta': {}}
'meta': {},
'sysmeta': {}}
self.assertEquals(container_info,
self.memcache.get(cache_key))
@ -366,7 +367,8 @@ class TestController(unittest.TestCase):
'container_count': None, # internally keep None
'total_object_count': None,
'bytes': None,
'meta': {}}
'meta': {},
'sysmeta': {}}
self.assertEquals(account_info,
self.memcache.get(cache_key))