openstack
/
swift
Code Issues Proposed changes

Enhance log msg to report referer and user-agent 

Enhance internally logged messages to report referer and user-agent.

Pass the referering URL and METHOD between internal servers (when
known), and set the user-agent to be the server type (obj-server,
container-server, proxy-server, obj-updater, obj-replicator,
container-updater, direct-client, etc.) with the process PID. In
conjunction with the transaction ID, it helps to track down which PID
from a given system was responsible for initiating the request and
what that server was working on to make this request.

This has been helpful in tracking down interactions between object,
container and account servers.

We also take things a bit further performaing a bit of refactoring to
consolidate calls to transfer_headers() now that we have a helper
method for constructing them.

Finally we performed further changes to avoid header key duplication
due to string literal header key values and the various objects
representing headers for requests and responses. See below for more
details.

====

Header Keys

There seems to be a bit of a problem with the case of the various
string literals used for header keys and the interchangable way
standard Python dictionaries, HeaderKeyDict() and HeaderEnvironProxy()
objects are used.

If one is not careful, a header object of some sort (one that does not
normalize its keys, and that is not necessarily a dictionary) can be
constructed containing header keys which differ only by the case of
their string literals. E.g.:

   { 'x-trans-id': '1234', 'X-Trans-Id': '5678' }

Such an object, when passed to http_connect() will result in an
on-the-wire header where the key values are merged together, comma
separated, that looks something like:

   HTTP_X_TRANS_ID: 1234,5678

For some headers in some contexts, this is behavior is desirable. For
example, one can also use a list of tuples which enumerate the multiple
values a single header should have.

However, in almost all of the contexts used in the code base, this is
not desirable.

This behavior arises from a combination of factors:

   1. Header strings are not constants and different lower-case and
      title-case header strings values are used interchangably in the
      code at times

      It might be worth the effort to make a pass through the code to
      stop using string literals and use constants instead, but there
      are plusses and minuses to doing that, so this was not attempted
      in this effort

   2. HeaderEnvironProxy() objects report their keys in ".title()"
      case, but normalize all other key references to the form
      expected by the Request class's environ field

      swob.Request.headers fields are HeaderEnvironProxy() objects.

   3. HeaderKeyDict() objects report their keys in ".lower()" case,
      and normalize all other key references to ".lower()" case

      swob.Response.headers fields are HeaderKeyDict() objects.

Depending on which object is used and how it is used, one can end up
with such a mismatch.

This commit takes the following steps as a (PROPOSED) solution:

   1. Change HeaderKeyDict() to normalize using ".title()" case to
      match HeaderEnvironProxy()

   2. Replace standard python dictionary objects with HeaderKeyDict()
      objects where possible

      This gives us an object that normalizes key references to avoid
      fixing the code to normalize the string literals.

   3. Fix up a few places to use title case string literals to match
      the new defaults

Change-Id: Ied56a1df83ffac793ee85e796424d7d20f18f469
Signed-off-by: Peter Portante <peter.portante@redhat.com>
This commit is contained in:
Peter Portante 2012-11-15 16:34:45 -05:00 committed by Gerrit Code Review
parent 7d625f6ea4
commit 8825c9c74a
20 changed files with 459 additions and 274 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
swift/common/direct_client.py
View File

@ -18,6 +18,7 @@ Internal client library for making calls directly to the servers rather than
through the proxy.
"""
import os
import socket
from httplib import HTTPException
from time import time
@ -30,6 +31,7 @@ from swiftclient import ClientException, json_loads
from swift.common.utils import normalize_timestamp
from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \
is_success, is_server_error
from swift.common.swob import HeaderKeyDict
def quote(value, safe='/'):
@ -38,6 +40,14 @@ def quote(value, safe='/'):
return _quote(value, safe)
def gen_headers(hdrs_in=None, add_ts=False):
hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict()
if add_ts:
hdrs_out['X-Timestamp'] = normalize_timestamp(time())
hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid()
return hdrs_out
def direct_get_account(node, part, account, marker=None, limit=None,
prefix=None, delimiter=None, conn_timeout=5,
response_timeout=15):
@ -68,7 +78,8 @@ def direct_get_account(node, part, account, marker=None, limit=None,
qs += '&delimiter=%s' % quote(delimiter)
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'GET', path, query_string=qs)
'GET', path, query_string=qs,
headers=gen_headers())
with Timeout(response_timeout):
resp = conn.getresponse()
if not is_success(resp.status):
@ -107,7 +118,7 @@ def direct_head_container(node, part, account, container, conn_timeout=5,
path = '/%s/%s' % (account, container)
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'HEAD', path)
'HEAD', path, headers=gen_headers())
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
@ -157,7 +168,8 @@ def direct_get_container(node, part, account, container, marker=None,
qs += '&delimiter=%s' % quote(delimiter)
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'GET', path, query_string=qs)
'GET', path, query_string=qs,
headers=gen_headers())
with Timeout(response_timeout):
resp = conn.getresponse()
if not is_success(resp.status):
@ -185,10 +197,10 @@ def direct_delete_container(node, part, account, container, conn_timeout=5,
headers = {}
path = '/%s/%s' % (account, container)
headers['X-Timestamp'] = normalize_timestamp(time())
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'DELETE', path, headers)
'DELETE', path,
headers=gen_headers(headers, True))
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
@ -220,7 +232,7 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5,
path = '/%s/%s/%s' % (account, container, obj)
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'HEAD', path)
'HEAD', path, headers=gen_headers())
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
@ -262,7 +274,7 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
path = '/%s/%s/%s' % (account, container, obj)
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'GET', path, headers=headers)
'GET', path, headers=gen_headers(headers))
with Timeout(response_timeout):
resp = conn.getresponse()
if not is_success(resp.status):
@ -328,10 +340,9 @@ def direct_put_object(node, part, account, container, name, contents,
headers['Content-Length'] = '0'
if isinstance(contents, basestring):
contents = [contents]
headers['X-Timestamp'] = normalize_timestamp(time())
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'PUT', path, headers=headers)
'PUT', path, headers=gen_headers(headers, True))
for chunk in contents:
conn.send(chunk)
with Timeout(response_timeout):
@ -365,10 +376,9 @@ def direct_post_object(node, part, account, container, name, headers,
:raises ClientException: HTTP POST request failed
"""
path = '/%s/%s/%s' % (account, container, name)
headers['X-Timestamp'] = normalize_timestamp(time())
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'POST', path, headers=headers)
'POST', path, headers=gen_headers(headers, True))
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
@ -401,10 +411,9 @@ def direct_delete_object(node, part, account, container, obj,
headers = {}
path = '/%s/%s/%s' % (account, container, obj)
headers['X-Timestamp'] = normalize_timestamp(time())
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'DELETE', path, headers)
'DELETE', path, headers=gen_headers(headers, True))
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()

4
swift/common/middleware/catch_errors.py
View File

@ -39,13 +39,13 @@ class CatchErrorsContext(WSGIContext):
resp = HTTPServerError(request=Request(env),
body='An error occurred',
content_type='text/plain')
resp.headers['x-trans-id'] = trans_id
resp.headers['X-Trans-Id'] = trans_id
return resp(env, start_response)
# make sure the response has the trans_id
if self._response_headers is None:
self._response_headers = []
self._response_headers.append(('x-trans-id', trans_id))
self._response_headers.append(('X-Trans-Id', trans_id))
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return resp

7
swift/common/middleware/tempurl.py
View File

@ -103,6 +103,7 @@ from urllib import urlencode
from urlparse import parse_qs
from swift.common.wsgi import make_pre_authed_env
from swift.common.swob import HeaderKeyDict
#: Default headers to remove from incoming requests. Simply a whitespace
@ -211,7 +212,7 @@ class TempURL(object):
headers = DEFAULT_OUTGOING_REMOVE_HEADERS
if 'outgoing_remove_headers' in conf:
headers = conf['outgoing_remove_headers']
headers = [h.lower() for h in headers.split()]
headers = [h.title() for h in headers.split()]
#: Headers to remove from outgoing responses. Lowercase, like
#: `x-account-meta-temp-url-key`.
self.outgoing_remove_headers = [h for h in headers if h[-1] != '*']
@ -223,7 +224,7 @@ class TempURL(object):
headers = DEFAULT_OUTGOING_ALLOW_HEADERS
if 'outgoing_allow_headers' in conf:
headers = conf['outgoing_allow_headers']
headers = [h.lower() for h in headers.split()]
headers = [h.title() for h in headers.split()]
#: Headers to allow in outgoing responses. Lowercase, like
#: `x-matches-remove-prefix-but-okay`.
self.outgoing_allow_headers = [h for h in headers if h[-1] != '*']
@ -474,7 +475,7 @@ class TempURL(object):
removed as per the middlware configuration for
outgoing responses.
"""
headers = dict(headers)
headers = HeaderKeyDict(headers)
for h in headers.keys():
remove = h in self.outgoing_remove_headers
if not remove:

23
swift/common/swob.py
View File

@ -231,7 +231,7 @@ class HeaderEnvironProxy(UserDict.DictMixin):
class HeaderKeyDict(dict):
"""
A dict that lower-cases all keys on the way in, so as to be
A dict that title-cases all keys on the way in, so as to be
case-insensitive.
"""
def __init__(self, *args, **kwargs):
@ -242,30 +242,30 @@ class HeaderKeyDict(dict):
def update(self, other):
if hasattr(other, 'keys'):
for key in other.keys():
self[key.lower()] = other[key]
self[key.title()] = other[key]
else:
for key, value in other:
self[key.lower()] = value
self[key.title()] = value
def __getitem__(self, key):
return dict.get(self, key.lower())
return dict.get(self, key.title())
def __setitem__(self, key, value):
if value is None:
self.pop(key.lower(), None)
self.pop(key.title(), None)
elif isinstance(value, unicode):
return dict.__setitem__(self, key.lower(), value.encode('utf-8'))
return dict.__setitem__(self, key.title(), value.encode('utf-8'))
else:
return dict.__setitem__(self, key.lower(), str(value))
return dict.__setitem__(self, key.title(), str(value))
def __contains__(self, key):
return dict.__contains__(self, key.lower())
return dict.__contains__(self, key.title())
def __delitem__(self, key):
return dict.__delitem__(self, key.lower())
return dict.__delitem__(self, key.title())
def get(self, key, default=None):
return dict.get(self, key.lower(), default)
return dict.get(self, key.title(), default)
def _resp_status_property():
@ -803,6 +803,9 @@ class Request(object):
"Provides the full url of the request"
return self.host_url + self.path_qs
def as_referer(self):
return self.method + ' ' + self.url
def path_info_pop(self):
"""
Takes one path portion (delineated by slashes) from the

8
swift/container/server.py
View File

@ -37,7 +37,7 @@ from swift.common.http import HTTP_NOT_FOUND, is_success
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPConflict, \
HTTPCreated, HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
HTTPPreconditionFailed, HTTPMethodNotAllowed, Request, Response, \
HTTPInsufficientStorage, HTTPNotAcceptable
HTTPInsufficientStorage, HTTPNotAcceptable, HeaderKeyDict
DATADIR = 'containers'
@ -126,12 +126,14 @@ class ContainerController(object):
account_ip, account_port = account_host.rsplit(':', 1)
new_path = '/' + '/'.join([account, container])
info = broker.get_info()
account_headers = {
account_headers = HeaderKeyDict({
'x-put-timestamp': info['put_timestamp'],
'x-delete-timestamp': info['delete_timestamp'],
'x-object-count': info['object_count'],
'x-bytes-used': info['bytes_used'],
'x-trans-id': req.headers.get('x-trans-id', '-')}
'x-trans-id': req.headers.get('x-trans-id', '-'),
'user-agent': 'container-server %s' % os.getpid(),
'referer': req.as_referer()})
if req.headers.get('x-account-override-deleted', 'no').lower() == \
'yes':
account_headers['x-account-override-deleted'] = 'yes'

15
swift/container/updater.py
View File

@ -61,6 +61,7 @@ class ContainerUpdater(Daemon):
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = os.path.join(self.recon_cache_path, "container.recon")
self.user_agent = 'container-updater %s' % os.getpid()
def get_account_ring(self):
"""Get the account ring. Load it if it hasn't been yet."""
@ -269,14 +270,16 @@ class ContainerUpdater(Daemon):
"""
with ConnectionTimeout(self.conn_timeout):
try:
headers = {
'X-Put-Timestamp': put_timestamp,
'X-Delete-Timestamp': delete_timestamp,
'X-Object-Count': count,
'X-Bytes-Used': bytes,
'X-Account-Override-Deleted': 'yes',
'user-agent': self.user_agent}
conn = http_connect(
node['ip'], node['port'], node['device'], part,
'PUT', container,
headers={'X-Put-Timestamp': put_timestamp,
'X-Delete-Timestamp': delete_timestamp,
'X-Object-Count': count,
'X-Bytes-Used': bytes,
'X-Account-Override-Deleted': 'yes'})
'PUT', container, headers=headers)
except (Exception, Timeout):
self.logger.exception(_(
'ERROR account update failed with '

19
swift/obj/replicator.py
View File

@ -271,6 +271,9 @@ class ObjectReplicator(Daemon):
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = os.path.join(self.recon_cache_path, "object.recon")
self.headers = {
'Content-Length': '0',
'user-agent': 'obj-replicator %s' % os.getpid()}
def _rsync(self, args):
"""
@ -389,12 +392,12 @@ class ObjectReplicator(Daemon):
success = self.rsync(node, job, suffixes)
if success:
with Timeout(self.http_timeout):
http_connect(
node['ip'], node['port'],
node['device'], job['partition'], 'REPLICATE',
'/' + '-'.join(suffixes),
headers={'Content-Length': '0'}).\
getresponse().read()
conn = http_connect(node['ip'], node['port'],
node['device'],
job['partition'], 'REPLICATE',
'/' + '-'.join(suffixes),
headers=self.headers)
conn.getresponse().read()
responses.append(success)
if not suffixes or (len(responses) ==
len(job['nodes']) and all(responses)):
@ -435,7 +438,7 @@ class ObjectReplicator(Daemon):
resp = http_connect(
node['ip'], node['port'],
node['device'], job['partition'], 'REPLICATE',
'', headers={'Content-Length': '0'}).getresponse()
'', headers=self.headers).getresponse()
if resp.status == HTTP_INSUFFICIENT_STORAGE:
self.logger.error(_('%(ip)s/%(device)s responded'
' as unmounted'), node)
@ -469,7 +472,7 @@ class ObjectReplicator(Daemon):
node['ip'], node['port'],
node['device'], job['partition'], 'REPLICATE',
'/' + '-'.join(suffixes),
headers={'Content-Length': '0'})
headers=self.headers)
conn.getresponse().read()
self.suffix_sync += len(suffixes)
self.logger.update_stats('suffix.syncs', len(suffixes))

51
swift/obj/server.py
View File

@ -46,7 +46,8 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
HTTPInternalServerError, HTTPNoContent, HTTPNotFound, HTTPNotModified, \
HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, UTC, \
HTTPInsufficientStorage, HTTPForbidden, multi_range_iterator
HTTPInsufficientStorage, HTTPForbidden, multi_range_iterator, \
HeaderKeyDict
DATADIR = 'objects'
@ -474,6 +475,7 @@ class ObjectController(object):
request
:param objdevice: device name that the object is in
"""
headers_out['user-agent'] = 'obj-server %s' % os.getpid()
full_path = '/%s/%s/%s' % (account, container, obj)
if all([host, partition, contdevice]):
try:
@ -508,7 +510,7 @@ class ObjectController(object):
normalize_timestamp(headers_out['x-timestamp'])),
os.path.join(self.devices, objdevice, 'tmp'))
def container_update(self, op, account, container, obj, headers_in,
def container_update(self, op, account, container, obj, request,
headers_out, objdevice):
"""
Update the container when objects are updated.
@ -517,11 +519,12 @@ class ObjectController(object):
:param account: account name for the object
:param container: container name for the object
:param obj: object name
:param headers_in: dictionary of headers from the original request
:param request: the original request object driving the update
:param headers_out: dictionary of headers to send in the container
request(s)
:param objdevice: device name that the object is in
"""
headers_in = request.headers
conthosts = [h.strip() for h in
headers_in.get('X-Container-Host', '').split(',')]
contdevices = [d.strip() for d in
@ -543,13 +546,15 @@ class ObjectController(object):
else:
updates = []
headers_out['x-trans-id'] = headers_in.get('x-trans-id', '-')
headers_out['referer'] = request.as_referer()
for conthost, contdevice in updates:
self.async_update(op, account, container, obj, conthost,
contpartition, contdevice, headers_out,
objdevice)
def delete_at_update(self, op, delete_at, account, container, obj,
headers_in, objdevice):
request, objdevice):
"""
Update the expiring objects container when objects are updated.
@ -557,7 +562,7 @@ class ObjectController(object):
:param account: account name for the object
:param container: container name for the object
:param obj: object name
:param headers_in: dictionary of headers from the original request
:param request: the original request driving the update
:param objdevice: device name that the object is in
"""
# Quick cap that will work from now until Sat Nov 20 17:46:39 2286
@ -568,8 +573,11 @@ class ObjectController(object):
partition = None
hosts = contdevices = [None]
headers_out = {'x-timestamp': headers_in['x-timestamp'],
'x-trans-id': headers_in.get('x-trans-id', '-')}
headers_in = request.headers
headers_out = HeaderKeyDict({
'x-timestamp': headers_in['x-timestamp'],
'x-trans-id': headers_in.get('x-trans-id', '-'),
'referer': request.as_referer()})
if op != 'DELETE':
partition = headers_in.get('X-Delete-At-Partition', None)
hosts = headers_in.get('X-Delete-At-Host', '')
@ -626,7 +634,7 @@ class ObjectController(object):
return HTTPNotFound(request=request)
metadata = {'X-Timestamp': request.headers['x-timestamp']}
metadata.update(val for val in request.headers.iteritems()
if val[0].lower().startswith('x-object-meta-'))
if val[0].startswith('X-Object-Meta-'))
for header_key in self.allowed_headers:
if header_key in request.headers:
header_caps = header_key.title()
@ -635,10 +643,10 @@ class ObjectController(object):
if old_delete_at != new_delete_at:
if new_delete_at:
self.delete_at_update('PUT', new_delete_at, account, container,
obj, request.headers, device)
obj, request, device)
if old_delete_at:
self.delete_at_update('DELETE', old_delete_at, account,
container, obj, request.headers, device)
container, obj, request, device)
disk_file.put_metadata(metadata)
return HTTPAccepted(request=request)
@ -728,11 +736,11 @@ class ObjectController(object):
if new_delete_at:
self.delete_at_update(
'PUT', new_delete_at, account, container, obj,
request.headers, device)
request, device)
if old_delete_at:
self.delete_at_update(
'DELETE', old_delete_at, account, container, obj,
request.headers, device)
request, device)
disk_file.put(fd, upload_size, metadata)
except DiskFileNoSpace:
return HTTPInsufficientStorage(drive=device, request=request)
@ -740,12 +748,12 @@ class ObjectController(object):
if not orig_timestamp or \
orig_timestamp < request.headers['x-timestamp']:
self.container_update(
'PUT', account, container, obj, request.headers,
{'x-size': disk_file.metadata['Content-Length'],
'x-content-type': disk_file.metadata['Content-Type'],
'x-timestamp': disk_file.metadata['X-Timestamp'],
'x-etag': disk_file.metadata['ETag'],
'x-trans-id': request.headers.get('x-trans-id', '-')},
'PUT', account, container, obj, request,
HeaderKeyDict({
'x-size': disk_file.metadata['Content-Length'],
'x-content-type': disk_file.metadata['Content-Type'],
'x-timestamp': disk_file.metadata['X-Timestamp'],
'x-etag': disk_file.metadata['ETag']}),
device)
resp = HTTPCreated(request=request, etag=etag)
return resp
@ -907,15 +915,14 @@ class ObjectController(object):
old_delete_at = int(disk_file.metadata.get('X-Delete-At') or 0)
if old_delete_at:
self.delete_at_update('DELETE', old_delete_at, account,
container, obj, request.headers, device)
container, obj, request, device)
disk_file.put_metadata(metadata, tombstone=True)
disk_file.unlinkold(metadata['X-Timestamp'])
if not orig_timestamp or \
orig_timestamp < request.headers['x-timestamp']:
self.container_update(
'DELETE', account, container, obj, request.headers,
{'x-timestamp': metadata['X-Timestamp'],
'x-trans-id': request.headers.get('x-trans-id', '-')},
'DELETE', account, container, obj, request,
HeaderKeyDict({'x-timestamp': metadata['X-Timestamp']}),
device)
resp = response_class(request=request)
return resp

4
swift/obj/updater.py
View File

@ -228,10 +228,12 @@ class ObjectUpdater(Daemon):
:param obj: object name being updated
:param headers: headers to send with the update
"""
headers_out = headers.copy()
headers_out['user-agent'] = 'obj-updater %s' % os.getpid()
try:
with ConnectionTimeout(self.conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'],
part, op, obj, headers)
part, op, obj, headers_out)
with Timeout(self.node_timeout):
resp = conn.getresponse()
resp.read()

21
swift/proxy/controllers/account.py
View File

@ -24,10 +24,9 @@
# These shenanigans are to ensure all related objects can be garbage
# collected. We've seen objects hang around forever otherwise.
import time
from urllib import unquote
from swift.common.utils import normalize_timestamp, public
from swift.common.utils import public
from swift.common.constraints import check_metadata, MAX_ACCOUNT_NAME_LENGTH
from swift.common.http import is_success, HTTP_NOT_FOUND
from swift.proxy.controllers.base import Controller, get_account_memcache_key
@ -57,9 +56,7 @@ class AccountController(Controller):
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id,
'Connection': 'close'}
headers = self.generate_request_headers(req)
resp = self.make_requests(
Request.blank('/v1/' + self.account_name),
self.app.account_ring, partition, 'PUT',
@ -90,10 +87,7 @@ class AccountController(Controller):
return resp
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'x-trans-id': self.trans_id,
'Connection': 'close'}
self.transfer_headers(req.headers, headers)
headers = self.generate_request_headers(req, transfer=True)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
@ -110,10 +104,7 @@ class AccountController(Controller):
return error_response
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id,
'Connection': 'close'}
self.transfer_headers(req.headers, headers)
headers = self.generate_request_headers(req, transfer=True)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
@ -150,9 +141,7 @@ class AccountController(Controller):
headers={'Allow': ', '.join(self.allowed_methods)})
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id,
'Connection': 'close'}
headers = self.generate_request_headers(req)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))

56
swift/proxy/controllers/base.py
View File

@ -24,6 +24,7 @@
# These shenanigans are to ensure all related objects can be garbage
# collected. We've seen objects hang around forever otherwise.
import os
import time
import functools
import inspect
@ -42,7 +43,7 @@ from swift.common.http import is_informational, is_success, is_redirection, \
is_server_error, HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_MULTIPLE_CHOICES, \
HTTP_BAD_REQUEST, HTTP_NOT_FOUND, HTTP_SERVICE_UNAVAILABLE, \
HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED
from swift.common.swob import Request, Response
from swift.common.swob import Request, Response, HeaderKeyDict
def update_headers(response, headers):
@ -178,8 +179,8 @@ def cors_validation(func):
'content-type', 'expires', 'last-modified',
'pragma', 'etag', 'x-timestamp', 'x-trans-id']
for header in resp.headers:
if header.startswith('x-container-meta') or \
header.startswith('x-object-meta'):
if header.startswith('X-Container-Meta') or \
header.startswith('X-Object-Meta'):
expose_headers.append(header.lower())
if cors_info.get('expose_headers'):
expose_headers.extend(
@ -280,20 +281,39 @@ class Controller(object):
return []
def transfer_headers(self, src_headers, dst_headers):
st = self.server_type.lower()
x_remove = 'x-remove-%s-meta-' % st
x_meta = 'x-%s-meta-' % st
dst_headers.update((k.lower().replace('-remove', '', 1), '')
for k in src_headers
if k.lower().startswith(x_remove) or
k.lower() in self._x_remove_headers())
x_meta = 'x-%s-meta-' % st
dst_headers.update((k.lower(), v)
for k, v in src_headers.iteritems()
if k.lower() in self.pass_through_headers or
k.lower().startswith(x_meta))
def generate_request_headers(self, orig_req=None, additional=None,
transfer=False):
# Use the additional headers first so they don't overwrite the headers
# we require.
headers = HeaderKeyDict(additional) if additional else HeaderKeyDict()
if transfer:
self.transfer_headers(orig_req.headers, headers)
if 'x-timestamp' not in headers:
headers['x-timestamp'] = normalize_timestamp(time.time())
if orig_req:
referer = orig_req.as_referer()
else:
referer = ''
headers.update({'x-trans-id': self.trans_id,
'connection': 'close',
'user-agent': 'proxy-server %s' % os.getpid(),
'referer': referer})
return headers
def error_occurred(self, node, msg):
"""
Handle logging, and handling of errors.
@ -359,11 +379,14 @@ class Controller(object):
{'msg': msg, 'ip': node['ip'],
'port': node['port'], 'device': node['device']})
def account_info(self, account, autocreate=False):
def account_info(self, account, req=None, autocreate=False):
"""
Get account information, and also verify that the account exists.
:param account: name of the account to get the info for
:param req: caller's HTTP request context object (optional)
:param autocreate: whether or not to automatically create the given
account or not (optional, default: False)
:returns: tuple of (account partition, account nodes, container_count)
or (None, None, None) if it does not exist
"""
@ -392,7 +415,7 @@ class Controller(object):
return None, None, None
result_code = 0
path = '/%s' % account
headers = {'x-trans-id': self.trans_id, 'Connection': 'close'}
headers = self.generate_request_headers(req)
for node in self.iter_nodes(self.app.account_ring, partition):
try:
start_node_timing = time.time()
@ -432,9 +455,7 @@ class Controller(object):
if result_code == HTTP_NOT_FOUND and autocreate:
if len(account) > MAX_ACCOUNT_NAME_LENGTH:
return None, None, None
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id,
'Connection': 'close'}
headers = self.generate_request_headers(req)
resp = self.make_requests(Request.blank('/v1' + path),
self.app.account_ring, partition, 'PUT',
path, [headers] * len(nodes))
@ -460,7 +481,8 @@ class Controller(object):
return partition, nodes, container_count
return None, None, None
def container_info(self, account, container, account_autocreate=False):
def container_info(self, account, container, req=None,
account_autocreate=False):
"""
Get container information and thusly verify container existence.
This will also make a call to account_info to verify that the
@ -468,6 +490,9 @@ class Controller(object):
:param account: account name for the container
:param container: container name to look up
:param req: caller's HTTP request context object (optional)
:param account_autocreate: whether or not to automatically create the
given account or not (optional, default: False)
:returns: dict containing at least container partition ('partition'),
container nodes ('containers'), container read
acl ('read_acl'), container write acl ('write_acl'),
@ -492,9 +517,10 @@ class Controller(object):
container_info['partition'] = part
container_info['nodes'] = nodes
return container_info
if not self.account_info(account, autocreate=account_autocreate)[1]:
if not self.account_info(account, req,
autocreate=account_autocreate)[1]:
return container_info
headers = {'x-trans-id': self.trans_id, 'Connection': 'close'}
headers = self.generate_request_headers(req)
for node in self.iter_nodes(self.app.container_ring, part):
try:
start_node_timing = time.time()
@ -784,8 +810,8 @@ class Controller(object):
start_node_timing = time.time()
try:
with ConnectionTimeout(self.app.conn_timeout):
headers = dict(req.headers)
headers['Connection'] = 'close'
headers = self.generate_request_headers(
req, additional=req.headers)
conn = http_connect(
node['ip'], node['port'], node['device'], partition,
req.method, path, headers=headers,

23
swift/proxy/controllers/container.py
View File

@ -24,10 +24,9 @@
# These shenanigans are to ensure all related objects can be garbage
# collected. We've seen objects hang around forever otherwise.
import time
from urllib import unquote
from swift.common.utils import normalize_timestamp, public, csv_append
from swift.common.utils import public, csv_append
from swift.common.constraints import check_metadata, MAX_CONTAINER_NAME_LENGTH
from swift.common.http import HTTP_ACCEPTED
from swift.proxy.controllers.base import Controller, delay_denial, \
@ -70,7 +69,7 @@ class ContainerController(Controller):
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
if not self.account_info(self.account_name)[1]:
if not self.account_info(self.account_name, req)[1]:
return HTTPNotFound(request=req)
part = self.app.container_ring.get_part(
self.account_name, self.container_name)
@ -125,7 +124,7 @@ class ContainerController(Controller):
(len(self.container_name), MAX_CONTAINER_NAME_LENGTH)
return resp
account_partition, accounts, container_count = \
self.account_info(self.account_name,
self.account_info(self.account_name, req,
autocreate=self.app.account_autocreate)
if self.app.max_containers_per_account > 0 and \
container_count >= self.app.max_containers_per_account and \
@ -158,16 +157,13 @@ class ContainerController(Controller):
if error_response:
return error_response
account_partition, accounts, container_count = \
self.account_info(self.account_name,
self.account_info(self.account_name, req,
autocreate=self.app.account_autocreate)
if not accounts:
return HTTPNotFound(request=req)
container_partition, containers = self.app.container_ring.get_nodes(
self.account_name, self.container_name)
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'x-trans-id': self.trans_id,
'Connection': 'close'}
self.transfer_headers(req.headers, headers)
headers = self.generate_request_headers(req, transfer=True)
if self.app.memcache:
self.app.memcache.delete(get_container_memcache_key(
self.account_name, self.container_name))
@ -181,7 +177,7 @@ class ContainerController(Controller):
def DELETE(self, req):
"""HTTP DELETE request handler."""
account_partition, accounts, container_count = \
self.account_info(self.account_name)
self.account_info(self.account_name, req)
if not accounts:
return HTTPNotFound(request=req)
container_partition, containers = self.app.container_ring.get_nodes(
@ -202,14 +198,9 @@ class ContainerController(Controller):
def _backend_requests(self, req, n_outgoing,
account_partition, accounts):
headers = [{'Connection': 'close',
'X-Timestamp': normalize_timestamp(time.time()),
'x-trans-id': self.trans_id}
headers = [self.generate_request_headers(req, transfer=True)
for _junk in range(n_outgoing)]
for header in headers:
self.transfer_headers(req.headers, header)
for i, account in enumerate(accounts):
i = i % len(headers)

18
swift/proxy/controllers/obj.py
View File

@ -377,8 +377,8 @@ class ObjectController(Controller):
def GETorHEAD(self, req):
"""Handle HTTP GET or HEAD requests."""
container_info = self.container_info(self.account_name,
self.container_name)
container_info = self.container_info(
self.account_name, self.container_name, req)
req.acl = container_info['read_acl']
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
@ -568,7 +568,7 @@ class ObjectController(Controller):
if error_response:
return error_response
container_info = self.container_info(
self.account_name, self.container_name,
self.account_name, self.container_name, req,
account_autocreate=self.app.account_autocreate)
container_partition = container_info['partition']
containers = container_info['nodes']
@ -614,7 +614,7 @@ class ObjectController(Controller):
def _backend_requests(self, req, n_outgoing,
container_partition, containers,
delete_at_partition=None, delete_at_nodes=None):
headers = [dict(req.headers.iteritems())
headers = [self.generate_request_headers(req, additional=req.headers)
for _junk in range(n_outgoing)]
for header in headers:
@ -692,7 +692,7 @@ class ObjectController(Controller):
def PUT(self, req):
"""HTTP PUT request handler."""
container_info = self.container_info(
self.account_name, self.container_name,
self.account_name, self.container_name, req,
account_autocreate=self.app.account_autocreate)
container_partition = container_info['partition']
containers = container_info['nodes']
@ -991,8 +991,8 @@ class ObjectController(Controller):
@delay_denial
def DELETE(self, req):
"""HTTP DELETE request handler."""
container_info = self.container_info(self.account_name,
self.container_name)
container_info = self.container_info(
self.account_name, self.container_name, req)
container_partition = container_info['partition']
containers = container_info['nodes']
req.acl = container_info['write_acl']
@ -1043,8 +1043,8 @@ class ObjectController(Controller):
self.container_name = lcontainer
self.object_name = last_item['name']
new_del_req = Request.blank(copy_path, environ=req.environ)
container_info = self.container_info(self.account_name,
self.container_name)
container_info = self.container_info(
self.account_name, self.container_name, req)
container_partition = container_info['partition']
containers = container_info['nodes']
new_del_req.acl = container_info['write_acl']

6
test/unit/common/middleware/test_except.py
View File

@ -59,7 +59,7 @@ class TestCatchErrors(unittest.TestCase):
self.assertEquals(self.logger.txn_id, None)
def start_response(status, headers, exc_info=None):
self.assert_('x-trans-id' in (x[0] for x in headers))
self.assert_('X-Trans-Id' in (x[0] for x in headers))
app = catch_errors.CatchErrorMiddleware(FakeApp(), {})
req = Request.blank('/v1/a/c/o')
app(req.environ, start_response)
@ -69,7 +69,7 @@ class TestCatchErrors(unittest.TestCase):
self.assertEquals(self.logger.txn_id, None)
def start_response(status, headers, exc_info=None):
self.assert_('x-trans-id' in (x[0] for x in headers))
self.assert_('X-Trans-Id' in (x[0] for x in headers))
app = catch_errors.CatchErrorMiddleware(FakeApp(True), {})
req = Request.blank('/v1/a/c/o')
app(req.environ, start_response)
@ -86,7 +86,7 @@ class TestCatchErrors(unittest.TestCase):
self.assertEquals(self.logger.txn_id, None)
def start_response(status, headers, exc_info=None):
self.assert_('x-trans-id' in (x[0] for x in headers))
self.assert_('X-Trans-Id' in (x[0] for x in headers))
app = catch_errors.CatchErrorMiddleware(
FakeApp(), {'trans_id_suffix': '-stuff'})
req = Request.blank('/v1/a/c/o')

12
test/unit/common/middleware/test_tempurl.py
View File

@ -19,7 +19,7 @@ from hashlib import sha1
from contextlib import contextmanager
from time import time
from swift.common.swob import Request, Response
from swift.common.swob import Request, Response, HeaderKeyDict
from swift.common.middleware import tempauth, tempurl
@ -706,7 +706,7 @@ class TestTempURL(unittest.TestCase):
orh = ''
oah = ''
hdrs = {'test-header': 'value'}
hdrs = dict(tempurl.TempURL(None,
hdrs = HeaderKeyDict(tempurl.TempURL(None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.iteritems()))
self.assertTrue('test-header' in hdrs)
@ -714,7 +714,7 @@ class TestTempURL(unittest.TestCase):
orh = 'test-header'
oah = ''
hdrs = {'test-header': 'value'}
hdrs = dict(tempurl.TempURL(None,
hdrs = HeaderKeyDict(tempurl.TempURL(None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.iteritems()))
self.assertTrue('test-header' not in hdrs)
@ -723,7 +723,7 @@ class TestTempURL(unittest.TestCase):
oah = ''
hdrs = {'test-header-one': 'value',
'test-header-two': 'value'}
hdrs = dict(tempurl.TempURL(None,
hdrs = HeaderKeyDict(tempurl.TempURL(None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.iteritems()))
self.assertTrue('test-header-one' not in hdrs)
@ -733,7 +733,7 @@ class TestTempURL(unittest.TestCase):
oah = 'test-header-two'
hdrs = {'test-header-one': 'value',
'test-header-two': 'value'}
hdrs = dict(tempurl.TempURL(None,
hdrs = HeaderKeyDict(tempurl.TempURL(None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.iteritems()))
self.assertTrue('test-header-one' not in hdrs)
@ -746,7 +746,7 @@ class TestTempURL(unittest.TestCase):
'test-other-header': 'value',
'test-header-yes': 'value',
'test-header-yes-this': 'value'}
hdrs = dict(tempurl.TempURL(None,
hdrs = HeaderKeyDict(tempurl.TempURL(None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.iteritems()))
self.assertTrue('test-header-one' not in hdrs)

32
test/unit/common/test_direct_client.py
View File

@ -16,14 +16,42 @@
# TODO: Tests
import unittest
import os
from swift.common import direct_client
class TestDirectClient(unittest.TestCase):
def test_placeholder(self):
pass
def test_quote(self):
res = direct_client.quote('123')
assert res == '123'
res = direct_client.quote('1&2&/3')
assert res == '1%262%26/3'
res = direct_client.quote('1&2&3', safe='&')
assert res == '1&2&3'
def test_gen_headers(self):
hdrs = direct_client.gen_headers()
assert 'user-agent' in hdrs
assert hdrs['user-agent'] == 'direct-client %s' % os.getpid()
assert len(hdrs.keys()) == 1
hdrs = direct_client.gen_headers(add_ts=True)
assert 'user-agent' in hdrs
assert 'x-timestamp' in hdrs
assert len(hdrs.keys()) == 2
hdrs = direct_client.gen_headers(hdrs_in={'foo-bar': '47'})
assert 'user-agent' in hdrs
assert 'foo-bar' in hdrs
assert hdrs['foo-bar'] == '47'
assert len(hdrs.keys()) == 2
hdrs = direct_client.gen_headers(hdrs_in={'user-agent': '47'})
assert 'user-agent' in hdrs
assert hdrs['user-agent'] == 'direct-client %s' % os.getpid()
assert len(hdrs.keys()) == 1
if __name__ == '__main__':

68
test/unit/common/test_swob.py
View File

@ -102,6 +102,15 @@ class TestHeaderKeyDict(unittest.TestCase):
self.assertEquals(headers.get('something-else'), None)
self.assertEquals(headers.get('something-else', True), True)
def test_keys(self):
headers = swift.common.swob.HeaderKeyDict()
headers['content-length'] = 20
headers['cOnTent-tYpe'] = 'text/plain'
headers['SomeThing-eLse'] = 'somevalue'
self.assertEquals(
set(headers.keys()),
set(('Content-Length', 'Content-Type', 'Something-Else')))
class TestRange(unittest.TestCase):
def test_range(self):
@ -502,6 +511,61 @@ class TestRequest(unittest.TestCase):
req.query_string = u'x=\u2661'
self.assertEquals(req.params['x'], u'\u2661'.encode('utf-8'))
def test_url(self):
pi = '/hi/there'
path = pi
req = swift.common.swob.Request.blank(path)
sche = 'http'
exp_url = '%(sche)s://localhost%(pi)s' % locals()
self.assertEqual(req.url, exp_url)
qs = 'hello=equal&acl'
path = '%s?%s' % (pi, qs)
s, p = 'unit.test.example.com', '90'
req = swift.common.swob.Request({'PATH_INFO': pi,
'QUERY_STRING': qs,
'SERVER_NAME': s,
'SERVER_PORT': p})
exp_url = '%(sche)s://%(s)s:%(p)s%(pi)s?%(qs)s' % locals()
self.assertEqual(req.url, exp_url)
host = 'unit.test.example.com'
req = swift.common.swob.Request({'PATH_INFO': pi,
'QUERY_STRING': qs,
'HTTP_HOST': host + ':80'})
exp_url = '%(sche)s://%(host)s%(pi)s?%(qs)s' % locals()
self.assertEqual(req.url, exp_url)
host = 'unit.test.example.com'
sche = 'https'
req = swift.common.swob.Request({'PATH_INFO': pi,
'QUERY_STRING': qs,
'HTTP_HOST': host + ':443',
'wsgi.url_scheme': sche})
exp_url = '%(sche)s://%(host)s%(pi)s?%(qs)s' % locals()
self.assertEqual(req.url, exp_url)
host = 'unit.test.example.com:81'
req = swift.common.swob.Request({'PATH_INFO': pi,
'QUERY_STRING': qs,
'HTTP_HOST': host,
'wsgi.url_scheme': sche})
exp_url = '%(sche)s://%(host)s%(pi)s?%(qs)s' % locals()
self.assertEqual(req.url, exp_url)
def test_as_referer(self):
pi = '/hi/there'
qs = 'hello=equal&acl'
sche = 'https'
host = 'unit.test.example.com:81'
req = swift.common.swob.Request({'REQUEST_METHOD': 'POST',
'PATH_INFO': pi,
'QUERY_STRING': qs,
'HTTP_HOST': host,
'wsgi.url_scheme': sche})
exp_url = '%(sche)s://%(host)s%(pi)s?%(qs)s' % locals()
self.assertEqual(req.as_referer(), 'POST ' + exp_url)
class TestStatusMap(unittest.TestCase):
def test_status_map(self):
@ -518,8 +582,8 @@ class TestStatusMap(unittest.TestCase):
self.assert_('The resource could not be found.' in body)
self.assertEquals(response_args[0], '404 Not Found')
headers = dict(response_args[1])
self.assertEquals(headers['content-type'], 'text/html; charset=UTF-8')
self.assert_(int(headers['content-length']) > 0)
self.assertEquals(headers['Content-Type'], 'text/html; charset=UTF-8')
self.assert_(int(headers['Content-Length']) > 0)
class TestResponse(unittest.TestCase):

14
test/unit/container/test_server.py
View File

@ -24,7 +24,7 @@ from tempfile import mkdtemp
from eventlet import spawn, Timeout, listen
import simplejson
from swift.common.swob import Request
from swift.common.swob import Request, HeaderKeyDict
import swift.container
from swift.container import server as container_server
from swift.common.utils import normalize_timestamp, mkdirs
@ -1353,11 +1353,13 @@ class TestContainerController(unittest.TestCase):
'partition': '30',
'method': 'PUT',
'ssl': False,
'headers': {'x-bytes-used': 0,
'headers': HeaderKeyDict({'x-bytes-used': 0,
'x-delete-timestamp': '0',
'x-object-count': 0,
'x-put-timestamp': '0000012345.00000',
'x-trans-id': '-'}})
'referer': 'PUT http://localhost/sda1/p/a/c',
'user-agent': 'container-server %d' % os.getpid(),
'x-trans-id': '-'})})
self.assertEquals(
http_connect_args[1],
{'ipaddr': '6.7.8.9',
@ -1367,11 +1369,13 @@ class TestContainerController(unittest.TestCase):
'partition': '30',
'method': 'PUT',
'ssl': False,
'headers': {'x-bytes-used': 0,
'headers': HeaderKeyDict({'x-bytes-used': 0,
'x-delete-timestamp': '0',
'x-object-count': 0,
'x-put-timestamp': '0000012345.00000',
'x-trans-id': '-'}})
'referer': 'PUT http://localhost/sda1/p/a/c',
'user-agent': 'container-server %d' % os.getpid(),
'x-trans-id': '-'})})
if __name__ == '__main__':

193
test/unit/obj/test_server.py
View File

@ -37,7 +37,7 @@ from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \
from swift.common.exceptions import DiskFileNotExist
from swift.common import constraints
from eventlet import tpool
from swift.common.swob import Request
from swift.common.swob import Request, HeaderKeyDict
class TestDiskFile(unittest.TestCase):
@ -1705,7 +1705,8 @@ class TestObjectController(unittest.TestCase):
finally:
object_server.http_connect = orig_http_connect
self.assertEquals(given_args, ['127.0.0.1', '1234', 'sdc1', 1, 'PUT',
'/a/c/o', {'x-timestamp': '1', 'x-out': 'set'}])
'/a/c/o', {'x-timestamp': '1', 'x-out': 'set',
'user-agent': 'obj-server %s' % os.getpid()}])
def test_updating_multiple_delete_at_container_servers(self):
@ -1771,11 +1772,13 @@ class TestObjectController(unittest.TestCase):
'partition': '20',
'method': 'PUT',
'ssl': False,
'headers': {'x-content-type': 'application/burrito',
'headers': HeaderKeyDict({'x-content-type': 'application/burrito',
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-size': '0',
'x-timestamp': '12345',
'x-trans-id': '-'}})
'referer': 'PUT http://localhost/sda1/p/a/c/o',
'user-agent': 'obj-server %d' % os.getpid(),
'x-trans-id': '-'})})
self.assertEquals(
http_connect_args[1],
{'ipaddr': '10.1.1.1',
@ -1785,11 +1788,13 @@ class TestObjectController(unittest.TestCase):
'partition': '6237',
'method': 'PUT',
'ssl': False,
'headers': {'x-content-type': 'text/plain',
'headers': HeaderKeyDict({'x-content-type': 'text/plain',
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-size': '0',
'x-timestamp': '12345',
'x-trans-id': '-'}})
'referer': 'PUT http://localhost/sda1/p/a/c/o',
'user-agent': 'obj-server %d' % os.getpid(),
'x-trans-id': '-'})})
self.assertEquals(
http_connect_args[2],
{'ipaddr': '10.2.2.2',
@ -1799,11 +1804,13 @@ class TestObjectController(unittest.TestCase):
'partition': '6237',
'method': 'PUT',
'ssl': False,
'headers': {'x-content-type': 'text/plain',
'headers': HeaderKeyDict({'x-content-type': 'text/plain',
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-size': '0',
'x-timestamp': '12345',
'x-trans-id': '-'}})
'referer': 'PUT http://localhost/sda1/p/a/c/o',
'user-agent': 'obj-server %d' % os.getpid(),
'x-trans-id': '-'})})
def test_updating_multiple_container_servers(self):
http_connect_args = []
@ -1858,11 +1865,13 @@ class TestObjectController(unittest.TestCase):
'partition': '20',
'method': 'PUT',
'ssl': False,
'headers': {'x-content-type': 'application/burrito',
'headers': HeaderKeyDict({'x-content-type': 'application/burrito',
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-size': '0',
'x-timestamp': '12345',
'x-trans-id': '-'}})
'referer': 'PUT http://localhost/sda1/p/a/c/o',
'user-agent': 'obj-server %d' % os.getpid(),
'x-trans-id': '-'})})
self.assertEquals(
http_connect_args[1],
{'ipaddr': '6.7.8.9',
@ -1872,11 +1881,13 @@ class TestObjectController(unittest.TestCase):
'partition': '20',
'method': 'PUT',
'ssl': False,
'headers': {'x-content-type': 'application/burrito',
'headers': HeaderKeyDict({'x-content-type': 'application/burrito',
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-size': '0',
'x-timestamp': '12345',
'x-trans-id': '-'}})
'referer': 'PUT http://localhost/sda1/p/a/c/o',
'user-agent': 'obj-server %d' % os.getpid(),
'x-trans-id': '-'})})
def test_async_update_saves_on_exception(self):
_prefix = utils.HASH_PATH_PREFIX
@ -1898,8 +1909,9 @@ class TestObjectController(unittest.TestCase):
pickle.load(open(os.path.join(self.testdir, 'sda1',
'async_pending', 'a83',
'06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))),
{'headers': {'x-timestamp': '1', 'x-out': 'set'}, 'account': 'a',
'container': 'c', 'obj': 'o', 'op': 'PUT'})
{'headers': {'x-timestamp': '1', 'x-out': 'set',
'user-agent': 'obj-server %s' % os.getpid()},
'account': 'a', 'container': 'c', 'obj': 'o', 'op': 'PUT'})
def test_async_update_saves_on_non_2xx(self):
_prefix = utils.HASH_PATH_PREFIX
@ -1931,7 +1943,8 @@ class TestObjectController(unittest.TestCase):
pickle.load(open(os.path.join(self.testdir, 'sda1',
'async_pending', 'a83',
'06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))),
{'headers': {'x-timestamp': '1', 'x-out': str(status)},
{'headers': {'x-timestamp': '1', 'x-out': str(status),
'user-agent': 'obj-server %s' % os.getpid()},
'account': 'a', 'container': 'c', 'obj': 'o',
'op': 'PUT'})
finally:
@ -1969,6 +1982,46 @@ class TestObjectController(unittest.TestCase):
finally:
object_server.http_connect = orig_http_connect
def test_container_update_no_async_update(self):
given_args = []
def fake_async_update(*args):
given_args.extend(args)
self.object_controller.async_update = fake_async_update
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': 1,
'X-Trans-Id': '1234'})
self.object_controller.container_update('PUT', 'a', 'c', 'o', req,
{'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-content-type': 'text/plain', 'x-timestamp': '1'}, 'sda1')
self.assertEquals(given_args, [])
def test_container_update(self):
given_args = []
def fake_async_update(*args):
given_args.extend(args)
self.object_controller.async_update = fake_async_update
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': 1,
'X-Trans-Id': '123',
'X-Container-Host': 'chost',
'X-Container-Partition': 'cpartition',
'X-Container-Device': 'cdevice'})
self.object_controller.container_update('PUT', 'a', 'c', 'o', req,
{'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-content-type': 'text/plain', 'x-timestamp': '1'}, 'sda1')
self.assertEquals(given_args, ['PUT', 'a', 'c', 'o', 'chost',
'cpartition', 'cdevice',
{'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-content-type': 'text/plain', 'x-timestamp': '1',
'x-trans-id': '123', 'referer': 'PUT http://localhost/v1/a/c/o'},
'sda1'])
def test_delete_at_update_put(self):
given_args = []
@ -1976,13 +2029,18 @@ class TestObjectController(unittest.TestCase):
given_args.extend(args)
self.object_controller.async_update = fake_async_update
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': 1,
'X-Trans-Id': '123'})
self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o',
{'x-timestamp': '1'}, 'sda1')
req, 'sda1')
self.assertEquals(given_args, ['PUT', '.expiring_objects', '0',
'2-a/c/o', None, None, None,
{'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
HeaderKeyDict({'x-size': '0',
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-content-type': 'text/plain', 'x-timestamp': '1',
'x-trans-id': '-'},
'x-trans-id': '123', 'referer': 'PUT http://localhost/v1/a/c/o'}),
'sda1'])
def test_delete_at_negative(self):
@ -1993,13 +2051,18 @@ class TestObjectController(unittest.TestCase):
given_args.extend(args)
self.object_controller.async_update = fake_async_update
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': 1,
'X-Trans-Id': '1234'})
self.object_controller.delete_at_update(
'PUT', -2, 'a', 'c', 'o', {'x-timestamp': '1'}, 'sda1')
'PUT', -2, 'a', 'c', 'o', req, 'sda1')
self.assertEquals(given_args, [
'PUT', '.expiring_objects', '0', '0-a/c/o', None, None, None,
{'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
HeaderKeyDict({'x-size': '0',
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-content-type': 'text/plain', 'x-timestamp': '1',
'x-trans-id': '-'},
'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}),
'sda1'])
def test_delete_at_cap(self):
@ -2010,14 +2073,19 @@ class TestObjectController(unittest.TestCase):
given_args.extend(args)
self.object_controller.async_update = fake_async_update
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': 1,
'X-Trans-Id': '1234'})
self.object_controller.delete_at_update(
'PUT', 12345678901, 'a', 'c', 'o', {'x-timestamp': '1'}, 'sda1')
'PUT', 12345678901, 'a', 'c', 'o', req, 'sda1')
self.assertEquals(given_args, [
'PUT', '.expiring_objects', '9999936000', '9999999999-a/c/o', None,
None, None,
{'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
HeaderKeyDict({'x-size': '0',
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-content-type': 'text/plain', 'x-timestamp': '1',
'x-trans-id': '-'},
'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}),
'sda1'])
def test_delete_at_update_put_with_info(self):
@ -2027,15 +2095,21 @@ class TestObjectController(unittest.TestCase):
given_args.extend(args)
self.object_controller.async_update = fake_async_update
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': 1,
'X-Trans-Id': '1234',
'X-Delete-At-Host': '127.0.0.1:1234',
'X-Delete-At-Partition': '3',
'X-Delete-At-Device': 'sdc1'})
self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o',
{'x-timestamp': '1', 'X-Delete-At-Host': '127.0.0.1:1234',
'X-Delete-At-Partition': '3', 'X-Delete-At-Device': 'sdc1'},
'sda1')
req, 'sda1')
self.assertEquals(given_args, ['PUT', '.expiring_objects', '0',
'2-a/c/o', '127.0.0.1:1234', '3', 'sdc1',
{'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
HeaderKeyDict({'x-size': '0',
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
'x-content-type': 'text/plain', 'x-timestamp': '1',
'x-trans-id': '-'},
'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}),
'sda1'])
def test_delete_at_update_delete(self):
@ -2045,11 +2119,16 @@ class TestObjectController(unittest.TestCase):
given_args.extend(args)
self.object_controller.async_update = fake_async_update
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'DELETE'},
headers={'X-Timestamp': 1,
'X-Trans-Id': '1234'})
self.object_controller.delete_at_update('DELETE', 2, 'a', 'c', 'o',
{'x-timestamp': '1'}, 'sda1')
req, 'sda1')
self.assertEquals(given_args, ['DELETE', '.expiring_objects', '0',
'2-a/c/o', None, None, None,
{'x-timestamp': '1', 'x-trans-id': '-'}, 'sda1'])
HeaderKeyDict({'x-timestamp': '1', 'x-trans-id': '1234',
'referer': 'DELETE http://localhost/v1/a/c/o'}), 'sda1'])
def test_POST_calls_delete_at(self):
given_args = []
@ -2089,11 +2168,7 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 202)
self.assertEquals(given_args, [
'PUT', int(delete_at_timestamp1), 'a', 'c', 'o',
{'X-Delete-At': delete_at_timestamp1,
'Content-Type': 'application/x-test',
'X-Timestamp': timestamp1,
'Host': 'localhost:80'},
'sda1'])
req, 'sda1'])
while given_args:
given_args.pop()
@ -2110,18 +2185,9 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 202)
self.assertEquals(given_args, [
'PUT', int(delete_at_timestamp2), 'a', 'c', 'o',
{'X-Delete-At': delete_at_timestamp2,
'Content-Type': 'application/x-test',
'X-Timestamp': timestamp2, 'Host': 'localhost:80'},
'sda1',
req, 'sda1',
'DELETE', int(delete_at_timestamp1), 'a', 'c', 'o',
# This 2 timestamp is okay because it's ignored since it's just
# part of the current request headers. The above 1 timestamp is the
# important one.
{'X-Delete-At': delete_at_timestamp2,
'Content-Type': 'application/x-test',
'X-Timestamp': timestamp2, 'Host': 'localhost:80'},
'sda1'])
req, 'sda1'])
def test_PUT_calls_delete_at(self):
given_args = []
@ -2153,12 +2219,7 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 201)
self.assertEquals(given_args, [
'PUT', int(delete_at_timestamp1), 'a', 'c', 'o',
{'X-Delete-At': delete_at_timestamp1,
'Content-Length': '4',
'Content-Type': 'application/octet-stream',
'X-Timestamp': timestamp1,
'Host': 'localhost:80'},
'sda1'])
req, 'sda1'])
while given_args:
given_args.pop()
@ -2177,20 +2238,9 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 201)
self.assertEquals(given_args, [
'PUT', int(delete_at_timestamp2), 'a', 'c', 'o',
{'X-Delete-At': delete_at_timestamp2,
'Content-Length': '4',
'Content-Type': 'application/octet-stream',
'X-Timestamp': timestamp2, 'Host': 'localhost:80'},
'sda1',
req, 'sda1',
'DELETE', int(delete_at_timestamp1), 'a', 'c', 'o',
# This 2 timestamp is okay because it's ignored since it's just
# part of the current request headers. The above 1 timestamp is the
# important one.
{'X-Delete-At': delete_at_timestamp2,
'Content-Length': '4',
'Content-Type': 'application/octet-stream',
'X-Timestamp': timestamp2, 'Host': 'localhost:80'},
'sda1'])
req, 'sda1'])
def test_GET_but_expired(self):
test_time = time() + 10000
@ -2434,12 +2484,7 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 201)
self.assertEquals(given_args, [
'PUT', int(delete_at_timestamp1), 'a', 'c', 'o',
{'X-Delete-At': delete_at_timestamp1,
'Content-Length': '4',
'Content-Type': 'application/octet-stream',
'X-Timestamp': timestamp1,
'Host': 'localhost:80'},
'sda1'])
req, 'sda1'])
while given_args:
given_args.pop()
@ -2454,9 +2499,7 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 204)
self.assertEquals(given_args, [
'DELETE', int(delete_at_timestamp1), 'a', 'c', 'o',
{'Content-Type': 'application/octet-stream',
'Host': 'localhost:80', 'X-Timestamp': timestamp2},
'sda1'])
req, 'sda1'])
def test_PUT_delete_at_in_past(self):
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},

124
test/unit/proxy/test_server.py
View File

@ -167,7 +167,7 @@ def setup():
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
assert(headers[:len(exp)] == exp)
assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)])
def teardown():
@ -311,8 +311,17 @@ class TestController(unittest.TestCase):
object_ring=FakeRing())
self.controller = swift.proxy.controllers.Controller(app)
class FakeReq(object):
def __init__(self):
self.url = "/foo/bar"
self.method = "METHOD"
def as_referer(self):
return self.method + ' ' + self.url
self.account = 'some_account'
self.container = 'some_container'
self.request = FakeReq()
self.read_acl = 'read_acl'
self.write_acl = 'write_acl'
@ -365,7 +374,7 @@ class TestController(unittest.TestCase):
with save_globals():
set_http_connect(200)
partition, nodes, count = \
self.controller.account_info(self.account)
self.controller.account_info(self.account, self.request)
set_http_connect(201, raise_timeout_exc=True)
self.controller._make_request(
nodes, partition, 'POST', '/', '', '',
@ -376,7 +385,7 @@ class TestController(unittest.TestCase):
with save_globals():
set_http_connect(200)
partition, nodes, count = \
self.controller.account_info(self.account)
self.controller.account_info(self.account, self.request)
self.check_account_info_return(partition, nodes)
self.assertEquals(count, 12345)
@ -391,7 +400,7 @@ class TestController(unittest.TestCase):
set_http_connect()
partition, nodes, count = \
self.controller.account_info(self.account)
self.controller.account_info(self.account, self.request)
self.check_account_info_return(partition, nodes)
self.assertEquals(count, 12345)
@ -400,7 +409,7 @@ class TestController(unittest.TestCase):
with save_globals():
set_http_connect(404, 404, 404)
partition, nodes, count = \
self.controller.account_info(self.account)
self.controller.account_info(self.account, self.request)
self.check_account_info_return(partition, nodes, True)
self.assertEquals(count, None)
@ -415,7 +424,7 @@ class TestController(unittest.TestCase):
set_http_connect()
partition, nodes, count = \
self.controller.account_info(self.account)
self.controller.account_info(self.account, self.request)
self.check_account_info_return(partition, nodes, True)
self.assertEquals(count, None)
@ -424,7 +433,7 @@ class TestController(unittest.TestCase):
def test(*status_list):
set_http_connect(*status_list)
partition, nodes, count = \
self.controller.account_info(self.account)
self.controller.account_info(self.account, self.request)
self.assertEqual(len(self.memcache.keys()), 0)
self.check_account_info_return(partition, nodes, True)
self.assertEquals(count, None)
@ -442,28 +451,28 @@ class TestController(unittest.TestCase):
# is True
set_http_connect(404, 404, 404)
partition, nodes, count = \
self.controller.account_info(self.account, autocreate=False)
self.controller.account_info(self.account, self.request, autocreate=False)
self.check_account_info_return(partition, nodes, is_none=True)
self.assertEquals(count, None)
self.memcache.store = {}
set_http_connect(404, 404, 404)
partition, nodes, count = \
self.controller.account_info(self.account)
self.controller.account_info(self.account, self.request)
self.check_account_info_return(partition, nodes, is_none=True)
self.assertEquals(count, None)
self.memcache.store = {}
set_http_connect(404, 404, 404, 201, 201, 201)
partition, nodes, count = \
self.controller.account_info(self.account, autocreate=True)
self.controller.account_info(self.account, self.request, autocreate=True)
self.check_account_info_return(partition, nodes)
self.assertEquals(count, 0)
self.memcache.store = {}
set_http_connect(404, 404, 404, 503, 201, 201)
partition, nodes, count = \
self.controller.account_info(self.account, autocreate=True)
self.controller.account_info(self.account, self.request, autocreate=True)
self.check_account_info_return(partition, nodes)
self.assertEquals(count, 0)
@ -471,7 +480,7 @@ class TestController(unittest.TestCase):
set_http_connect(404, 404, 404, 503, 201, 503)
exc = None
partition, nodes, count = \
self.controller.account_info(self.account, autocreate=True)
self.controller.account_info(self.account, self.request, autocreate=True)
self.check_account_info_return(partition, nodes, is_none=True)
self.assertEquals(None, count)
@ -479,7 +488,7 @@ class TestController(unittest.TestCase):
set_http_connect(404, 404, 404, 403, 403, 403)
exc = None
partition, nodes, count = \
self.controller.account_info(self.account, autocreate=True)
self.controller.account_info(self.account, self.request, autocreate=True)
self.check_account_info_return(partition, nodes, is_none=True)
self.assertEquals(None, count)
@ -487,7 +496,7 @@ class TestController(unittest.TestCase):
set_http_connect(404, 404, 404, 409, 409, 409)
exc = None
partition, nodes, count = \
self.controller.account_info(self.account, autocreate=True)
self.controller.account_info(self.account, self.request, autocreate=True)
self.check_account_info_return(partition, nodes, is_none=True)
self.assertEquals(None, count)
@ -504,18 +513,19 @@ class TestController(unittest.TestCase):
self.assertEqual(write_acl, ret['write_acl'])
def test_container_info_invalid_account(self):
def account_info(self, account, autocreate=False):
def account_info(self, account, request, autocreate=False):
return None, None
with save_globals():
swift.proxy.controllers.Controller.account_info = account_info
ret = self.controller.container_info(self.account,
self.container)
self.container,
self.request)
self.check_container_info_return(ret, True)
# tests if 200 is cached and used
def test_container_info_200(self):
def account_info(self, account, autocreate=False):
def account_info(self, account, request, autocreate=False):
return True, True, 0
with save_globals():
@ -523,8 +533,8 @@ class TestController(unittest.TestCase):
'x-container-write': self.write_acl}
swift.proxy.controllers.Controller.account_info = account_info
set_http_connect(200, headers=headers)
ret = self.controller.container_info(self.account,
self.container)
ret = self.controller.container_info(
self.account, self.container, self.request)
self.check_container_info_return(ret)
cache_key = get_container_memcache_key(self.account,
@ -534,20 +544,20 @@ class TestController(unittest.TestCase):
self.assertEquals(200, cache_value.get('status'))
set_http_connect()
ret = self.controller.container_info(self.account,
self.container)
ret = self.controller.container_info(
self.account, self.container, self.request)
self.check_container_info_return(ret)
# tests if 404 is cached and used
def test_container_info_404(self):
def account_info(self, account, autocreate=False):
def account_info(self, account, request, autocreate=False):
return True, True, 0
with save_globals():
swift.proxy.controllers.Controller.account_info = account_info
set_http_connect(404, 404, 404)
ret = self.controller.container_info(self.account,
self.container)
ret = self.controller.container_info(
self.account, self.container, self.request)
self.check_container_info_return(ret, True)
cache_key = get_container_memcache_key(self.account,
@ -557,16 +567,16 @@ class TestController(unittest.TestCase):
self.assertEquals(404, cache_value.get('status'))
set_http_connect()
ret = self.controller.container_info(self.account,
self.container)
ret = self.controller.container_info(
self.account, self.container, self.request)
self.check_container_info_return(ret, True)
# tests if some http status codes are not cached
def test_container_info_no_cache(self):
def test(*status_list):
set_http_connect(*status_list)
ret = self.controller.container_info(self.account,
self.container)
ret = self.controller.container_info(
self.account, self.container, self.request)
self.assertEqual(len(self.memcache.keys()), 0)
self.check_container_info_return(ret, True)
@ -4303,13 +4313,13 @@ class TestObjectController(unittest.TestCase):
200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT
self.assertEqual(seen_headers, [
{'X-Container-Host': '10.0.0.0:1000',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sda'},
{'X-Container-Host': '10.0.0.1:1001',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sdb'},
{'X-Container-Host': '10.0.0.2:1002',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sdc'}])
def test_PUT_x_container_headers_with_fewer_container_replicas(self):
@ -4324,10 +4334,10 @@ class TestObjectController(unittest.TestCase):
self.assertEqual(seen_headers, [
{'X-Container-Host': '10.0.0.0:1000',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sda'},
{'X-Container-Host': '10.0.0.1:1001',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sdb'},
{'X-Container-Host': None,
'X-Container-Partition': None,
@ -4345,13 +4355,13 @@ class TestObjectController(unittest.TestCase):
self.assertEqual(seen_headers, [
{'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sda,sdd'},
{'X-Container-Host': '10.0.0.1:1001',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sdb'},
{'X-Container-Host': '10.0.0.2:1002',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sdc'}])
def test_POST_x_container_headers_with_more_container_replicas(self):
@ -4367,13 +4377,13 @@ class TestObjectController(unittest.TestCase):
self.assertEqual(seen_headers, [
{'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sda,sdd'},
{'X-Container-Host': '10.0.0.1:1001',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sdb'},
{'X-Container-Host': '10.0.0.2:1002',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sdc'}])
def test_DELETE_x_container_headers_with_more_container_replicas(self):
@ -4388,13 +4398,13 @@ class TestObjectController(unittest.TestCase):
self.assertEqual(seen_headers, [
{'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sda,sdd'},
{'X-Container-Host': '10.0.0.1:1001',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sdb'},
{'X-Container-Host': '10.0.0.2:1002',
'X-Container-Partition': 1,
'X-Container-Partition': '1',
'X-Container-Device': 'sdc'}])
def test_PUT_x_delete_at_with_fewer_container_replicas(self):
@ -4413,10 +4423,10 @@ class TestObjectController(unittest.TestCase):
self.assertEqual(seen_headers, [
{'X-Delete-At-Host': '10.0.0.0:1000',
'X-Delete-At-Partition': 1,
'X-Delete-At-Partition': '1',
'X-Delete-At-Device': 'sda'},
{'X-Delete-At-Host': '10.0.0.1:1001',
'X-Delete-At-Partition': 1,
'X-Delete-At-Partition': '1',
'X-Delete-At-Device': 'sdb'},
{'X-Delete-At-Host': None,
'X-Delete-At-Partition': None,
@ -4439,13 +4449,13 @@ class TestObjectController(unittest.TestCase):
'X-Delete-At-Partition'))
self.assertEqual(seen_headers, [
{'X-Delete-At-Host': '10.0.0.0:1000,10.0.0.3:1003',
'X-Delete-At-Partition': 1,
'X-Delete-At-Partition': '1',
'X-Delete-At-Device': 'sda,sdd'},
{'X-Delete-At-Host': '10.0.0.1:1001',
'X-Delete-At-Partition': 1,
'X-Delete-At-Partition': '1',
'X-Delete-At-Device': 'sdb'},
{'X-Delete-At-Host': '10.0.0.2:1002',
'X-Delete-At-Partition': 1,
'X-Delete-At-Partition': '1',
'X-Delete-At-Device': 'sdc'}])
@ -5189,10 +5199,10 @@ class TestContainerController(unittest.TestCase):
200, 201, 201, 201) # HEAD PUT PUT PUT
self.assertEqual(seen_headers, [
{'X-Account-Host': '10.0.0.0:1000',
'X-Account-Partition': 1,
'X-Account-Partition': '1',
'X-Account-Device': 'sda'},
{'X-Account-Host': '10.0.0.1:1001',
'X-Account-Partition': 1,
'X-Account-Partition': '1',
'X-Account-Device': 'sdb'},
{'X-Account-Host': None,
'X-Account-Partition': None,
@ -5208,13 +5218,13 @@ class TestContainerController(unittest.TestCase):
200, 201, 201, 201) # HEAD PUT PUT PUT
self.assertEqual(seen_headers, [
{'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003',
'X-Account-Partition': 1,
'X-Account-Partition': '1',
'X-Account-Device': 'sda,sdd'},
{'X-Account-Host': '10.0.0.1:1001',
'X-Account-Partition': 1,
'X-Account-Partition': '1',
'X-Account-Device': 'sdb'},
{'X-Account-Host': '10.0.0.2:1002',
'X-Account-Partition': 1,
'X-Account-Partition': '1',
'X-Account-Device': 'sdc'}])
def test_DELETE_x_account_headers_with_fewer_account_replicas(self):
@ -5227,10 +5237,10 @@ class TestContainerController(unittest.TestCase):
200, 204, 204, 204) # HEAD DELETE DELETE DELETE
self.assertEqual(seen_headers, [
{'X-Account-Host': '10.0.0.0:1000',
'X-Account-Partition': 1,
'X-Account-Partition': '1',
'X-Account-Device': 'sda'},
{'X-Account-Host': '10.0.0.1:1001',
'X-Account-Partition': 1,
'X-Account-Partition': '1',
'X-Account-Device': 'sdb'},
{'X-Account-Host': None,
'X-Account-Partition': None,
@ -5246,13 +5256,13 @@ class TestContainerController(unittest.TestCase):
200, 204, 204, 204) # HEAD DELETE DELETE DELETE
self.assertEqual(seen_headers, [
{'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003',
'X-Account-Partition': 1,
'X-Account-Partition': '1',
'X-Account-Device': 'sda,sdd'},
{'X-Account-Host': '10.0.0.1:1001',
'X-Account-Partition': 1,
'X-Account-Partition': '1',
'X-Account-Device': 'sdb'},
{'X-Account-Host': '10.0.0.2:1002',
'X-Account-Partition': 1,
'X-Account-Partition': '1',
'X-Account-Device': 'sdc'}])