openstack/swift
Code Issues Proposed changes

Merge "Merge branch 'master' feature/crypto" into feature/crypto

Browse Source
This commit is contained in:
Jenkins 2016-05-26 06:47:08 +00:00 committed by Gerrit Code Review
commit b896240e9b
50 changed files with 1304 additions and 746 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
MANIFEST.in
View File

@ -6,7 +6,7 @@ include tox.ini
include requirements.txt test-requirements.txt
graft doc
graft etc
graft locale
graft swift/locale
graft test/functional
graft test/probe
graft test/unit

2
README.rst
View File

@ -33,7 +33,7 @@ Getting Started
Swift is part of OpenStack and follows the code contribution, review, and testing processes common to all OpenStack projects.
If you would like to start contributing, check out these
`notes <CONTRIBUTING.md>`__ to help you get started.
`notes <CONTRIBUTING.rst>`__ to help you get started.
The best place to get started is the
`"SAIO - Swift All In One" <http://docs.openstack.org/developer/swift/development_saio.html>`__.

1
doc/source/associated_projects.rst
View File

@ -107,7 +107,6 @@ Other
* `Glance <https://github.com/openstack/glance>`_ - Provides services for discovering, registering, and retrieving virtual machine images (for OpenStack Compute [Nova], for example).
* `Better Staticweb <https://github.com/CloudVPS/better-staticweb>`_ - Makes swift containers accessible by default.
* `Swiftsync <https://github.com/stackforge/swiftsync>`_ - A massive syncer between two swift clusters.
* `Django Swiftbrowser <https://github.com/cschwede/django-swiftbrowser>`_ - Simple Django web app to access OpenStack Swift.
* `Swift-account-stats <https://github.com/enovance/swift-account-stats>`_ - Swift-account-stats is a tool to report statistics on Swift usage at tenant and global levels.
* `PyECLib <https://bitbucket.org/kmgreen2/pyeclib>`_ - High Level Erasure Code library used by Swift

6
doc/source/development_guidelines.rst
View File

@ -27,6 +27,12 @@ To execute the tests:
pip install tox
* Generate list of distribution packages to install for testing::
tox -e bindep
Now install these packages using your distribution package manager
like apt-get, dnf, yum, or zypper.
* Run Tox from the root of the swift repo::

14
doc/source/development_saio.rst
View File

@ -591,3 +591,17 @@ doesn't work, here are some good starting places to look for issues:
you check that you can ``GET`` account, use ``sudo service memcached status``
and check if memcache is running. If memcache is not running, start it using
``sudo service memcached start``. Once memcache is running, rerun ``GET`` account.
------------
Known Issues
------------
Listed here are some "gotcha's" that you may run into when using or testing your SAIO:
#. fallocate_reserve - in most cases a SAIO doesn't have a very large XFS partition
so having fallocate enabled and fallocate_reserve set can cause issues, specifically
when trying to run the functional tests. For this reason fallocate has been turned
off on the object-servers in the SAIO. If you want to play with the fallocate_reserve
settings then know that functional tests will fail unless you change the max_file_size
constraint to something more reasonable then the default (5G). Ideally you'd make
it 1/4 of your XFS file system size so the tests can pass.

1
doc/source/getting_started.rst
View File

@ -15,6 +15,7 @@ Swift is written in Python and has these dependencies:
* rsync 3.0
* The Python packages listed in `the requirements file <https://github.com/openstack/swift/blob/master/requirements.txt>`_
* Testing additionally requires `the test dependencies <https://github.com/openstack/swift/blob/master/test-requirements.txt>`_
* Testing requires `these distribution packages <https://github.com/openstack/swift/blob/master/other-requirements.txt>`_
There is no current support for Python 3.

17
other-requirements.txt Normal file
View File

@ -0,0 +1,17 @@
# This is a cross-platform list tracking distribution packages needed by tests;
# see http://docs.openstack.org/infra/bindep/ for additional information.
build-essential [platform:dpkg]
gcc [platform:rpm]
gettext
liberasurecode-dev [platform:dpkg]
liberasurecode-devel [platform:rpm]
libffi-dev [platform:dpkg]
libffi-devel [platform:rpm]
memcached
python-dev [platform:dpkg]
python-devel [platform:rpm]
rsync
xfsprogs
libssl-dev [platform:dpkg]
openssl-devel [platform:rpm]

11
swift/common/container_sync_realms.py
View File

@ -57,7 +57,8 @@ class ContainerSyncRealms(object):
log_func = self.logger.debug
else:
log_func = self.logger.error
log_func(_('Could not load %r: %s'), self.conf_path, err)
log_func(_('Could not load %(conf)r: %(error)s') % {
'conf': self.conf_path, 'error': err})
else:
if mtime != self.conf_path_mtime:
self.conf_path_mtime = mtime
@ -66,7 +67,8 @@ class ContainerSyncRealms(object):
conf.read(self.conf_path)
except configparser.ParsingError as err:
self.logger.error(
_('Could not load %r: %s'), self.conf_path, err)
_('Could not load %(conf)r: %(error)s')
% {'conf': self.conf_path, 'error': err})
else:
try:
self.mtime_check_interval = conf.getint(
@ -79,8 +81,9 @@ class ContainerSyncRealms(object):
now + self.mtime_check_interval
except (configparser.ParsingError, ValueError) as err:
self.logger.error(
_('Error in %r with mtime_check_interval: %s'),
self.conf_path, err)
_('Error in %(conf)r with '
'mtime_check_interval: %(error)s')
% {'conf': self.conf_path, 'error': err})
realms = {}
for section in conf.sections():
realm = {}

3
swift/common/crypto_utils.py
View File

@ -67,7 +67,8 @@ class CryptoWSGIContext(WSGIContext):
self.logger.exception(_("Did not get a keys dict"))
except ValueError as e:
# don't include the key in any messages!
self.logger.exception(_("Bad key for %r: %s") % (name, str(e)))
self.logger.exception(_("Bad key for %(name)r: %(err)s") %
{'name': name, 'err': e})
raise HTTPInternalServerError(
"Unable to retrieve encryption keys.")

13
swift/common/db_replicator.py
View File

@ -525,10 +525,13 @@ class Replicator(Daemon):
if shouldbehere:
shouldbehere = bool([n for n in nodes if n['id'] == node_id])
# See Footnote [1] for an explanation of the repl_nodes assignment.
i = 0
while i < len(nodes) and nodes[i]['id'] != node_id:
i += 1
repl_nodes = nodes[i + 1:] + nodes[:i]
if len(nodes) > 1:
i = 0
while i < len(nodes) and nodes[i]['id'] != node_id:
i += 1
repl_nodes = nodes[i + 1:] + nodes[:i]
else: # Special case if using only a single replica
repl_nodes = nodes
more_nodes = self.ring.get_more_nodes(int(partition))
if not local_dev:
# Check further if local device is a handoff node
@ -563,7 +566,7 @@ class Replicator(Daemon):
except (Exception, Timeout):
self.logger.exception('UNHANDLED EXCEPTION: in post replicate '
'hook for %s', broker.db_file)
if not shouldbehere and all(responses):
if not shouldbehere and responses and all(responses):
# If the db shouldn't be on this node and has been successfully
# synced to all of its peers, it can be removed.
if not self.delete_db(broker):

6
swift/common/middleware/copy.py
View File

@ -102,7 +102,7 @@ accounts:
-------------------
Large Object Copy
-------------------
The best option to copy a large option is to copy segments individually.
The best option to copy a large object is to copy segments individually.
To copy the manifest object of a large object, add the query parameter to
the copy request::
@ -425,7 +425,9 @@ class ServerSideCopyMiddleware(object):
# Existing sys and user meta of source object is added to response
# headers in addition to the new ones.
for k, v in sink_req.headers.items():
if is_sys_or_user_meta('object', k) or k.lower() == 'x-delete-at':
if (is_sys_or_user_meta('object', k) or
k.lower() == 'x-delete-at' or
is_object_transient_sysmeta(k)):
resp_headers[k] = v
return resp_headers

3
swift/common/middleware/keystoneauth.py
View File

@ -287,7 +287,8 @@ class KeystoneAuth(object):
def _get_project_domain_id(self, environ):
info = get_account_info(environ, self.app, 'KS')
domain_id = info.get('sysmeta', {}).get('project-domain-id')
exists = is_success(info.get('status', 0))
exists = (is_success(info.get('status', 0))
and info.get('account_really_exists', True))
return exists, domain_id
def _set_project_domain_id(self, req, path_parts, env_identity):

1
swift/common/swob.py
View File

@ -1086,6 +1086,7 @@ class Response(object):
content_range = _header_property('content-range')
etag = _resp_etag_property()
status = _resp_status_property()
status_int = None
body = _resp_body_property()
host_url = _host_url_property()
last_modified = _datetime_property('last-modified')

28
swift/common/utils.py
View File

@ -596,7 +596,8 @@ class FileLikeIter(object):
class FallocateWrapper(object):
def __init__(self, noop=False):
if noop:
self.noop = noop
if self.noop:
self.func_name = 'posix_fallocate'
self.fallocate = noop_libc_function
return
@ -614,16 +615,18 @@ class FallocateWrapper(object):
def __call__(self, fd, mode, offset, length):
"""The length parameter must be a ctypes.c_uint64."""
if FALLOCATE_RESERVE > 0:
st = os.fstatvfs(fd)
free = st.f_frsize * st.f_bavail - length.value
if FALLOCATE_IS_PERCENT:
free = (float(free) / float(st.f_frsize * st.f_blocks)) * 100
if float(free) <= float(FALLOCATE_RESERVE):
raise OSError(
errno.ENOSPC,
'FALLOCATE_RESERVE fail %s <= %s' % (free,
FALLOCATE_RESERVE))
if not self.noop:
if FALLOCATE_RESERVE > 0:
st = os.fstatvfs(fd)
free = st.f_frsize * st.f_bavail - length.value
if FALLOCATE_IS_PERCENT:
free = \
(float(free) / float(st.f_frsize * st.f_blocks)) * 100
if float(free) <= float(FALLOCATE_RESERVE):
raise OSError(
errno.ENOSPC,
'FALLOCATE_RESERVE fail %s <= %s' %
(free, FALLOCATE_RESERVE))
args = {
'fallocate': (fd, mode, offset, length),
'posix_fallocate': (fd, offset, length)
@ -2671,7 +2674,8 @@ def validate_sync_to(value, allowed_sync_hosts, realms_conf):
endpoint = realms_conf.endpoint(realm, cluster)
if not endpoint:
return (
_('No cluster endpoint for %r %r') % (realm, cluster),
_('No cluster endpoint for %(realm)r %(cluster)r')
% {'realm': realm, 'cluster': cluster},
None, None, None)
return (
None,

9
swift/common/wsgi.py
View File

@ -196,9 +196,10 @@ def get_socket(conf):
raise
sleep(0.1)
if not sock:
raise Exception(_('Could not bind to %s:%s '
'after trying for %s seconds') % (
bind_addr[0], bind_addr[1], bind_timeout))
raise Exception(_('Could not bind to %(addr)s:%(port)s '
'after trying for %(timeout)s seconds') % {
'addr': bind_addr[0], 'port': bind_addr[1],
'timeout': bind_timeout})
# in my experience, sockets can hang around forever without keepalive
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
@ -1100,7 +1101,7 @@ def make_env(env, method=None, path=None, agent='Swift', query_string=None,
'swift.trans_id', 'swift.authorize_override',
'swift.authorize', 'HTTP_X_USER_ID', 'HTTP_X_PROJECT_ID',
'HTTP_REFERER', 'swift.orig_req_method', 'swift.log_info',
'swift.metadata.checked'):
'swift.infocache', 'swift.metadata.checked'):
if name in env:
newenv[name] = env[name]
if method:

11
swift/container/server.py
View File

@ -183,11 +183,12 @@ class ContainerController(BaseStorageServer):
if len(account_hosts) != len(account_devices):
# This shouldn't happen unless there's a bug in the proxy,
# but if there is, we want to know about it.
self.logger.error(_('ERROR Account update failed: different '
'numbers of hosts and devices in request: '
'"%s" vs "%s"') %
(req.headers.get('X-Account-Host', ''),
req.headers.get('X-Account-Device', '')))
self.logger.error(_(
'ERROR Account update failed: different '
'numbers of hosts and devices in request: '
'"%(hosts)s" vs "%(devices)s"') % {
'hosts': req.headers.get('X-Account-Host', ''),
'devices': req.headers.get('X-Account-Device', '')})
return HTTPBadRequest(req=req)
if account_partition:

5
swift/container/sync.py
View File

@ -237,8 +237,9 @@ class ContainerSync(Daemon):
if err.errno != errno.ENOENT:
raise
raise SystemExit(
_('Unable to load internal client from config: %r (%s)') %
(internal_client_conf_path, err))
_('Unable to load internal client from config: '
'%(conf)r (%(error)s)')
% {'conf': internal_client_conf_path, 'error': err})
def run_forever(self, *args, **kwargs):
"""

5
swift/obj/diskfile.py
View File

@ -378,8 +378,9 @@ def object_audit_location_generator(devices, mount_check=True, logger=None,
base, policy = split_policy_string(dir_)
except PolicyError as e:
if logger:
logger.warning(_('Directory %r does not map '
'to a valid policy (%s)') % (dir_, e))
logger.warning(_('Directory %(directory)r does not map '
'to a valid policy (%(error)s)') % {
'directory': dir_, 'error': e})
continue
datadir_path = os.path.join(devices, device, dir_)

19
swift/obj/expirer.py
View File

@ -77,15 +77,17 @@ class ObjectExpirer(Daemon):
"""
if final:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass completed in %ds; %d objects expired') %
(elapsed, self.report_objects))
self.logger.info(_('Pass completed in %(time)ds; '
'%(objects)d objects expired') % {
'time': elapsed, 'objects': self.report_objects})
dump_recon_cache({'object_expiration_pass': elapsed,
'expired_last_pass': self.report_objects},
self.rcache, self.logger)
elif time() - self.report_last_time >= self.report_interval:
elapsed = time() - self.report_first_time
self.logger.info(_('Pass so far %ds; %d objects expired') %
(elapsed, self.report_objects))
self.logger.info(_('Pass so far %(time)ds; '
'%(objects)d objects expired') % {
'time': elapsed, 'objects': self.report_objects})
self.report_last_time = time()
def iter_cont_objs_to_expire(self):
@ -168,8 +170,10 @@ class ObjectExpirer(Daemon):
self.logger.debug('Run begin')
containers, objects = \
self.swift.get_account_info(self.expiring_objects_account)
self.logger.info(_('Pass beginning; %s possible containers; %s '
'possible objects') % (containers, objects))
self.logger.info(_('Pass beginning; '
'%(containers)s possible containers; '
'%(objects)s possible objects') % {
'containers': containers, 'objects': objects})
for container, obj in self.iter_cont_objs_to_expire():
containers_to_delete.add(container)
@ -295,5 +299,6 @@ class ObjectExpirer(Daemon):
"""
path = '/v1/' + urllib.parse.quote(actual_obj.lstrip('/'))
self.swift.make_request('DELETE', path,
{'X-If-Delete-At': str(timestamp)},
{'X-If-Delete-At': str(timestamp),
'X-Timestamp': str(timestamp)},
(2, HTTP_PRECONDITION_FAILED))

3
swift/obj/reconstructor.py
View File

@ -843,6 +843,9 @@ class ObjectReconstructor(Daemon):
self.part_count += len(partitions)
for partition in partitions:
part_path = join(obj_path, partition)
if partition in ('auditor_status_ALL.json',
'auditor_status_ZBF.json'):
continue
if not (partition.isdigit() and
os.path.isdir(part_path)):
self.logger.warning(

11
swift/obj/server.py
View File

@ -282,11 +282,12 @@ class ObjectController(BaseStorageServer):
if len(conthosts) != len(contdevices):
# This shouldn't happen unless there's a bug in the proxy,
# but if there is, we want to know about it.
self.logger.error(_('ERROR Container update failed: different '
'numbers of hosts and devices in request: '
'"%s" vs "%s"') %
(headers_in.get('X-Container-Host', ''),
headers_in.get('X-Container-Device', '')))
self.logger.error(_(
'ERROR Container update failed: different '
'numbers of hosts and devices in request: '
'"%(hosts)s" vs "%(devices)s"') % {
'hosts': headers_in.get('X-Container-Host', ''),
'devices': headers_in.get('X-Container-Device', '')})
return
if contpartition:

6
swift/obj/updater.py
View File

@ -160,9 +160,9 @@ class ObjectUpdater(Daemon):
try:
base, policy = split_policy_string(asyncdir)
except PolicyError as e:
self.logger.warning(_('Directory %r does not map '
'to a valid policy (%s)') %
(asyncdir, e))
self.logger.warning(_('Directory %(directory)r does not map '
'to a valid policy (%(error)s)') % {
'directory': asyncdir, 'error': e})
continue
for prefix in self._listdir(async_pending):
prefix_path = os.path.join(async_pending, prefix)

26
swift/proxy/controllers/account.py
View File

@ -24,7 +24,8 @@ from swift.common.utils import public
from swift.common.constraints import check_metadata
from swift.common import constraints
from swift.common.http import HTTP_NOT_FOUND, HTTP_GONE
from swift.proxy.controllers.base import Controller, clear_info_cache
from swift.proxy.controllers.base import Controller, clear_info_cache, \
set_info_cache
from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed
from swift.common.request_helpers import get_sys_meta_prefix
@ -57,6 +58,9 @@ class AccountController(Controller):
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name),
constraints.MAX_ACCOUNT_NAME_LENGTH)
# Don't cache this. We know the account doesn't exist because
# the name is bad; we don't need to cache that because it's
# really cheap to recompute.
return resp
partition = self.app.account_ring.get_part(self.account_name)
@ -70,8 +74,28 @@ class AccountController(Controller):
if resp.headers.get('X-Account-Status', '').lower() == 'deleted':
resp.status = HTTP_GONE
elif self.app.account_autocreate:
# This is kind of a lie; we pretend like the account is
# there, but it's not. We'll create it as soon as something
# tries to write to it, but we don't need databases on disk
# to tell us that nothing's there.
#
# We set a header so that certain consumers can tell it's a
# fake listing. The important one is the PUT of a container
# to an autocreate account; the proxy checks to see if the
# account exists before actually performing the PUT and
# creates the account if necessary. If we feed it a perfect
# lie, it'll just try to create the container without
# creating the account, and that'll fail.
resp = account_listing_response(self.account_name, req,
get_listing_content_type(req))
resp.headers['X-Backend-Fake-Account-Listing'] = 'yes'
# Cache this. We just made a request to a storage node and got
# up-to-date information for the account.
resp.headers['X-Backend-Recheck-Account-Existence'] = str(
self.app.recheck_account_existence)
set_info_cache(self.app, req.environ, self.account_name, None, resp)
if req.environ.get('swift_owner'):
self.add_acls_from_sys_metadata(resp)
else:

432
swift/proxy/controllers/base.py
View File

@ -32,6 +32,7 @@ import functools
import inspect
import itertools
import operator
from copy import deepcopy
from sys import exc_info
from swift import gettext_ as _
@ -51,7 +52,7 @@ from swift.common.header_key_dict import HeaderKeyDict
from swift.common.http import is_informational, is_success, is_redirection, \
is_server_error, HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_MULTIPLE_CHOICES, \
HTTP_BAD_REQUEST, HTTP_NOT_FOUND, HTTP_SERVICE_UNAVAILABLE, \
HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED, HTTP_CONTINUE
HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED, HTTP_CONTINUE, HTTP_GONE
from swift.common.swob import Request, Response, Range, \
HTTPException, HTTPRequestedRangeNotSatisfiable, HTTPServiceUnavailable, \
status_map
@ -62,6 +63,10 @@ from swift.common.request_helpers import strip_sys_meta_prefix, \
from swift.common.storage_policy import POLICIES
DEFAULT_RECHECK_ACCOUNT_EXISTENCE = 60 # seconds
DEFAULT_RECHECK_CONTAINER_EXISTENCE = 60 # seconds
def update_headers(response, headers):
"""
Helper function to update headers in the response.
@ -103,18 +108,6 @@ def delay_denial(func):
return func
def get_account_memcache_key(account):
cache_key, env_key = _get_cache_key(account, None)
return cache_key
def get_container_memcache_key(account, container):
if not container:
raise ValueError("container not provided")
cache_key, env_key = _get_cache_key(account, container)
return cache_key
def _prep_headers_to_info(headers, server_type):
"""
Helper method that iterates once over a dict of headers,
@ -141,7 +134,7 @@ def headers_to_account_info(headers, status_int=HTTP_OK):
Construct a cacheable dict of account info based on response headers.
"""
headers, meta, sysmeta = _prep_headers_to_info(headers, 'account')
return {
account_info = {
'status': status_int,
# 'container_count' anomaly:
# Previous code sometimes expects an int sometimes a string
@ -151,8 +144,12 @@ def headers_to_account_info(headers, status_int=HTTP_OK):
'total_object_count': headers.get('x-account-object-count'),
'bytes': headers.get('x-account-bytes-used'),
'meta': meta,
'sysmeta': sysmeta
'sysmeta': sysmeta,
}
if is_success(status_int):
account_info['account_really_exists'] = not config_true_value(
headers.get('x-backend-fake-account-listing'))
return account_info
def headers_to_container_info(headers, status_int=HTTP_OK):
@ -175,7 +172,7 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
'max_age': meta.get('access-control-max-age')
},
'meta': meta,
'sysmeta': sysmeta
'sysmeta': sysmeta,
}
@ -287,8 +284,17 @@ def get_object_info(env, app, path=None, swift_source=None):
split_path(path or env['PATH_INFO'], 4, 4, True)
info = _get_object_info(app, env, account, container, obj,
swift_source=swift_source)
if not info:
if info:
info = deepcopy(info)
else:
info = headers_to_object_info({}, 0)
for field in ('length',):
if info.get(field) is None:
info[field] = 0
else:
info[field] = int(info[field])
return info
@ -304,11 +310,55 @@ def get_container_info(env, app, swift_source=None):
"""
(version, account, container, unused) = \
split_path(env['PATH_INFO'], 3, 4, True)
info = get_info(app, env, account, container, ret_not_found=True,
swift_source=swift_source)
# Check in environment cache and in memcache (in that order)
info = _get_info_from_caches(app, env, account, container)
if not info:
# Cache miss; go HEAD the container and populate the caches
env.setdefault('swift.infocache', {})
# Before checking the container, make sure the account exists.
#
# If it is an autocreateable account, just assume it exists; don't
# HEAD the account, as a GET or HEAD response for an autocreateable
# account is successful whether the account actually has .db files
# on disk or not.
is_autocreate_account = account.startswith(
getattr(app, 'auto_create_account_prefix', '.'))
if not is_autocreate_account:
account_info = get_account_info(env, app, swift_source)
if not account_info or not is_success(account_info['status']):
return headers_to_container_info({}, 0)
req = _prepare_pre_auth_info_request(
env, ("/%s/%s/%s" % (version, account, container)),
(swift_source or 'GET_CONTAINER_INFO'))
resp = req.get_response(app)
# Check in infocache to see if the proxy (or anyone else) already
# populated the cache for us. If they did, just use what's there.
#
# See similar comment in get_account_info() for justification.
info = _get_info_from_infocache(env, account, container)
if info is None:
info = set_info_cache(app, env, account, container, resp)
if info:
info = deepcopy(info) # avoid mutating what's in swift.infocache
else:
info = headers_to_container_info({}, 0)
# Old data format in memcache immediately after a Swift upgrade; clean
# it up so consumers of get_container_info() aren't exposed to it.
info.setdefault('storage_policy', '0')
if 'object_count' not in info and 'container_size' in info:
info['object_count'] = info.pop('container_size')
for field in ('bytes', 'object_count'):
if info.get(field) is None:
info[field] = 0
else:
info[field] = int(info[field])
return info
@ -322,32 +372,71 @@ def get_account_info(env, app, swift_source=None):
This call bypasses auth. Success does not imply that the request has
authorization to the account.
:raises ValueError: when path can't be split(path, 2, 4)
:raises ValueError: when path doesn't contain an account
"""
(version, account, _junk, _junk) = \
split_path(env['PATH_INFO'], 2, 4, True)
info = get_info(app, env, account, ret_not_found=True,
swift_source=swift_source)
# Check in environment cache and in memcache (in that order)
info = _get_info_from_caches(app, env, account)
# Cache miss; go HEAD the account and populate the caches
if not info:
info = headers_to_account_info({}, 0)
if info.get('container_count') is None:
info['container_count'] = 0
env.setdefault('swift.infocache', {})
req = _prepare_pre_auth_info_request(
env, "/%s/%s" % (version, account),
(swift_source or 'GET_ACCOUNT_INFO'))
resp = req.get_response(app)
# Check in infocache to see if the proxy (or anyone else) already
# populated the cache for us. If they did, just use what's there.
#
# The point of this is to avoid setting the value in memcached
# twice. Otherwise, we're needlessly sending requests across the
# network.
#
# If the info didn't make it into the cache, we'll compute it from
# the response and populate the cache ourselves.
#
# Note that this is taking "exists in infocache" to imply "exists in
# memcache". That's because we're trying to avoid superfluous
# network traffic, and checking in memcache prior to setting in
# memcache would defeat the purpose.
info = _get_info_from_infocache(env, account)
if info is None:
info = set_info_cache(app, env, account, None, resp)
if info:
info = info.copy() # avoid mutating what's in swift.infocache
else:
info['container_count'] = int(info['container_count'])
info = headers_to_account_info({}, 0)
for field in ('container_count', 'bytes', 'total_object_count'):
if info.get(field) is None:
info[field] = 0
else:
info[field] = int(info[field])
return info
def _get_cache_key(account, container):
def get_cache_key(account, container=None, obj=None):
"""
Get the keys for both memcache (cache_key) and env (env_key)
where info about accounts and containers is cached
Get the keys for both memcache and env['swift.infocache'] (cache_key)
where info about accounts, containers, and objects is cached
:param account: The name of the account
:param account: The name of the account
:param container: The name of the container (or None if account)
:returns: a tuple of (cache_key, env_key)
:param obj: The name of the object (or None if account or container)
:returns: a string cache_key
"""
if container:
if obj:
if not (account and container):
raise ValueError('Object cache key requires account and container')
cache_key = 'object/%s/%s/%s' % (account, container, obj)
elif container:
if not account:
raise ValueError('Container cache key requires account')
cache_key = 'container/%s/%s' % (account, container)
else:
cache_key = 'account/%s' % account
@ -355,58 +444,44 @@ def _get_cache_key(account, container):
# This allows caching both account and container and ensures that when we
# copy this env to form a new request, it won't accidentally reuse the
# old container or account info
env_key = 'swift.%s' % cache_key
return cache_key, env_key
return cache_key
def get_object_env_key(account, container, obj):
"""
Get the keys for env (env_key) where info about object is cached
:param account: The name of the account
:param container: The name of the container
:param obj: The name of the object
:returns: a string env_key
"""
env_key = 'swift.object/%s/%s/%s' % (account,
container, obj)
return env_key
def _set_info_cache(app, env, account, container, resp):
def set_info_cache(app, env, account, container, resp):
"""
Cache info in both memcache and env.
Caching is used to avoid unnecessary calls to account & container servers.
This is a private function that is being called by GETorHEAD_base and
by clear_info_cache.
Any attempt to GET or HEAD from the container/account server should use
the GETorHEAD_base interface which would than set the cache.
:param app: the application object
:param account: the unquoted account name
:param container: the unquoted container name or None
:param resp: the response received or None if info cache should be cleared
"""
:param resp: the response received or None if info cache should be cleared
if container:
cache_time = app.recheck_container_existence
else:
cache_time = app.recheck_account_existence
cache_key, env_key = _get_cache_key(account, container)
:returns: the info that was placed into the cache, or None if the
request status was not in (404, 410, 2xx).
"""
infocache = env.setdefault('swift.infocache', {})
cache_time = None
if container and resp:
cache_time = int(resp.headers.get(
'X-Backend-Recheck-Container-Existence',
DEFAULT_RECHECK_CONTAINER_EXISTENCE))
elif resp:
cache_time = int(resp.headers.get(
'X-Backend-Recheck-Account-Existence',
DEFAULT_RECHECK_ACCOUNT_EXISTENCE))
cache_key = get_cache_key(account, container)
if resp:
if resp.status_int == HTTP_NOT_FOUND:
if resp.status_int in (HTTP_NOT_FOUND, HTTP_GONE):
cache_time *= 0.1
elif not is_success(resp.status_int):
cache_time = None
else:
cache_time = None
# Next actually set both memcache and the env cache
memcache = getattr(app, 'memcache', None) or env.get('swift.cache')
if not cache_time:
env.pop(env_key, None)
infocache.pop(cache_key, None)
if memcache:
memcache.delete(cache_key)
return
@ -417,35 +492,35 @@ def _set_info_cache(app, env, account, container, resp):
info = headers_to_account_info(resp.headers, resp.status_int)
if memcache:
memcache.set(cache_key, info, time=cache_time)
env[env_key] = info
infocache[cache_key] = info
return info
def _set_object_info_cache(app, env, account, container, obj, resp):
def set_object_info_cache(app, env, account, container, obj, resp):
"""
Cache object info env. Do not cache object information in
memcache. This is an intentional omission as it would lead
to cache pressure. This is a per-request cache.
Caching is used to avoid unnecessary calls to object servers.
This is a private function that is being called by GETorHEAD_base.
Any attempt to GET or HEAD from the object server should use
the GETorHEAD_base interface which would then set the cache.
Cache object info in the WSGI environment, but not in memcache. Caching
in memcache would lead to cache pressure and mass evictions due to the
large number of objects in a typical Swift cluster. This is a
per-request cache only.
:param app: the application object
:param account: the unquoted account name
:param container: the unquoted container name or None
:param object: the unquoted object name or None
:param resp: the response received or None if info cache should be cleared
:param container: the unquoted container name
:param object: the unquoted object name
:param resp: a GET or HEAD response received from an object server, or
None if info cache should be cleared
:returns: the object info
"""
env_key = get_object_env_key(account, container, obj)
cache_key = get_cache_key(account, container, obj)
if not resp:
env.pop(env_key, None)
if 'swift.infocache' in env and not resp:
env['swift.infocache'].pop(cache_key, None)
return
info = headers_to_object_info(resp.headers, resp.status_int)
env[env_key] = info
env.setdefault('swift.infocache', {})[cache_key] = info
return info
def clear_info_cache(app, env, account, container=None):
@ -453,26 +528,43 @@ def clear_info_cache(app, env, account, container=None):
Clear the cached info in both memcache and env
:param app: the application object
:param env: the WSGI environment
:param account: the account name
:param container: the containr name or None if setting info for containers
"""
_set_info_cache(app, env, account, container, None)
set_info_cache(app, env, account, container, None)
def _get_info_cache(app, env, account, container=None):
def _get_info_from_infocache(env, account, container=None):
"""
Get the cached info from env or memcache (if used) in that order
Used for both account and container info
A private function used by get_info
Get cached account or container information from request-environment
cache (swift.infocache).
:param env: the environment used by the current request
:param account: the account name
:param container: the container name
:returns: a dictionary of cached info on cache hit, None on miss
"""
cache_key = get_cache_key(account, container)
if 'swift.infocache' in env and cache_key in env['swift.infocache']:
return env['swift.infocache'][cache_key]
return None
def _get_info_from_memcache(app, env, account, container=None):
"""
Get cached account or container information from memcache
:param app: the application object
:param env: the environment used by the current request
:returns: the cached info or None if not cached
"""
:param account: the account name
:param container: the container name
cache_key, env_key = _get_cache_key(account, container)
if env_key in env:
return env[env_key]
:returns: a dictionary of cached info on cache hit, None on miss. Also
returns None if memcache is not in use.
"""
cache_key = get_cache_key(account, container)
memcache = getattr(app, 'memcache', None) or env.get('swift.cache')
if memcache:
info = memcache.get(cache_key)
@ -480,15 +572,31 @@ def _get_info_cache(app, env, account, container=None):
for key in info:
if isinstance(info[key], six.text_type):
info[key] = info[key].encode("utf-8")
if isinstance(info[key], dict):
elif isinstance(info[key], dict):
for subkey, value in info[key].items():
if isinstance(value, six.text_type):
info[key][subkey] = value.encode("utf-8")
env[env_key] = info
env.setdefault('swift.infocache', {})[cache_key] = info
return info
return None
def _get_info_from_caches(app, env, account, container=None):
"""
Get the cached info from env or memcache (if used) in that order.
Used for both account and container info.
:param app: the application object
:param env: the environment used by the current request
:returns: the cached info or None if not cached
"""
info = _get_info_from_infocache(env, account, container)
if info is None:
info = _get_info_from_memcache(app, env, account, container)
return info
def _prepare_pre_auth_info_request(env, path, swift_source):
"""
Prepares a pre authed request to obtain info using a HEAD.
@ -504,14 +612,18 @@ def _prepare_pre_auth_info_request(env, path, swift_source):
# This is a sub request for container metadata- drop the Origin header from
# the request so the it is not treated as a CORS request.
newenv.pop('HTTP_ORIGIN', None)
# ACLs are only shown to account owners, so let's make sure this request
# looks like it came from the account owner.
newenv['swift_owner'] = True
# Note that Request.blank expects quoted path
return Request.blank(quote(path), environ=newenv)
def get_info(app, env, account, container=None, ret_not_found=False,
swift_source=None):
def get_info(app, env, account, container=None, swift_source=None):
"""
Get the info about accounts or containers
Get info about accounts or containers
Note: This call bypasses auth. Success does not imply that the
request has authorization to the info.
@ -520,37 +632,25 @@ def get_info(app, env, account, container=None, ret_not_found=False,
:param env: the environment used by the current request
:param account: The unquoted name of the account
:param container: The unquoted name of the container (or None if account)
:returns: the cached info or None if cannot be retrieved
:param swift_source: swift source logged for any subrequests made while
retrieving the account or container info
:returns: information about the specified entity in a dictionary. See
get_account_info and get_container_info for details on what's in the
dictionary.
"""
info = _get_info_cache(app, env, account, container)
if info:
if ret_not_found or is_success(info['status']):
return info
return None
# Not in cache, let's try the account servers
path = '/v1/%s' % account
if container:
# Stop and check if we have an account?
if not get_info(app, env, account) and not account.startswith(
getattr(app, 'auto_create_account_prefix', '.')):
return None
path += '/' + container
env.setdefault('swift.infocache', {})
req = _prepare_pre_auth_info_request(
env, path, (swift_source or 'GET_INFO'))
# Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in
# the environment under environ[env_key] and in memcache. We will
# pick the one from environ[env_key] and use it to set the caller env
resp = req.get_response(app)
cache_key, env_key = _get_cache_key(account, container)
try:
info = resp.environ[env_key]
env[env_key] = info
if ret_not_found or is_success(info['status']):
return info
except (KeyError, AttributeError):
pass
return None
if container:
path = '/v1/%s/%s' % (account, container)
path_env = env.copy()
path_env['PATH_INFO'] = path
return get_container_info(path_env, app, swift_source=swift_source)
else:
# account info
path = '/v1/%s' % (account,)
path_env = env.copy()
path_env['PATH_INFO'] = path
return get_account_info(path_env, app, swift_source=swift_source)
def _get_object_info(app, env, account, container, obj, swift_source=None):
@ -567,24 +667,22 @@ def _get_object_info(app, env, account, container, obj, swift_source=None):
:param obj: The unquoted name of the object
:returns: the cached info or None if cannot be retrieved
"""
env_key = get_object_env_key(account, container, obj)
info = env.get(env_key)
cache_key = get_cache_key(account, container, obj)
info = env.get('swift.infocache', {}).get(cache_key)
if info:
return info
# Not in cached, let's try the object servers
# Not in cache, let's try the object servers
path = '/v1/%s/%s/%s' % (account, container, obj)
req = _prepare_pre_auth_info_request(env, path, swift_source)
# Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in
# the environment under environ[env_key]. We will
# pick the one from environ[env_key] and use it to set the caller env
resp = req.get_response(app)
try:
info = resp.environ[env_key]
env[env_key] = info
return info
except (KeyError, AttributeError):
pass
return None
# Unlike get_account_info() and get_container_info(), we don't save
# things in memcache, so we can store the info without network traffic,
# *and* the proxy doesn't cache object info for us, so there's no chance
# that the object info would be in the environment. Thus, we just
# compute the object info based on the response and stash it in
# swift.infocache.
info = set_object_info_cache(app, env, account, container, obj, resp)
return info
def close_swift_conn(src):
@ -683,21 +781,21 @@ class ResumingGetter(object):
if begin is None:
# this is a -50 range req (last 50 bytes of file)
end -= num_bytes
if end == 0:
# we sent out exactly the first range's worth of bytes, so
# we're done with it
raise RangeAlreadyComplete()
else:
begin += num_bytes
if end and begin == end + 1:
# we sent out exactly the first range's worth of bytes, so
# we're done with it
raise RangeAlreadyComplete()
elif end and begin > end:
raise HTTPRequestedRangeNotSatisfiable()
elif end and begin:
req_range.ranges = [(begin, end)] + req_range.ranges[1:]
elif end:
req_range.ranges = [(None, end)] + req_range.ranges[1:]
else:
req_range.ranges = [(begin, None)] + req_range.ranges[1:]
if end is not None and begin == end + 1:
# we sent out exactly the first range's worth of bytes, so
# we're done with it
raise RangeAlreadyComplete()
if end is not None and (begin > end or end < 0):
raise HTTPRequestedRangeNotSatisfiable()
req_range.ranges = [(begin, end)] + req_range.ranges[1:]
self.backend_headers['Range'] = str(req_range)
else:
self.backend_headers['Range'] = 'bytes=%d-' % num_bytes
@ -1355,8 +1453,14 @@ class Controller(object):
env = getattr(req, 'environ', {})
else:
env = {}
info = get_info(self.app, env, account)
if not info:
env.setdefault('swift.infocache', {})
path_env = env.copy()
path_env['PATH_INFO'] = "/v1/%s" % (account,)
info = get_account_info(path_env, self.app)
if (not info
or not is_success(info['status'])
or not info.get('account_really_exists', True)):
return None, None, None
if info.get('container_count') is None:
container_count = 0
@ -1383,8 +1487,11 @@ class Controller(object):
env = getattr(req, 'environ', {})
else:
env = {}
info = get_info(self.app, env, account, container)
if not info:
env.setdefault('swift.infocache', {})
path_env = env.copy()
path_env['PATH_INFO'] = "/v1/%s/%s" % (account, container)
info = get_container_info(path_env, self.app)
if not info or not is_success(info.get('status')):
info = headers_to_container_info({}, 0)
info['partition'] = None
info['nodes'] = None
@ -1672,17 +1779,7 @@ class Controller(object):
req, handler.statuses, handler.reasons, handler.bodies,
'%s %s' % (server_type, req.method),
headers=handler.source_headers)
try:
(vrs, account, container) = req.split_path(2, 3)
_set_info_cache(self.app, req.environ, account, container, res)
except ValueError:
pass
try:
(vrs, account, container, obj) = req.split_path(4, 4, True)
_set_object_info_cache(self.app, req.environ, account,
container, obj, res)
except ValueError:
pass
# if a backend policy index is present in resp headers, translate it
# here with the friendly policy name
if 'X-Backend-Storage-Policy-Index' in res.headers and \
@ -1697,6 +1794,7 @@ class Controller(object):
'Could not translate %s (%r) from %r to policy',
'X-Backend-Storage-Policy-Index',
res.headers['X-Backend-Storage-Policy-Index'], path)
return res
def is_origin_allowed(self, cors_info, origin):

17
swift/proxy/controllers/container.py
View File

@ -22,7 +22,7 @@ from swift.common.constraints import check_metadata
from swift.common import constraints
from swift.common.http import HTTP_ACCEPTED, is_success
from swift.proxy.controllers.base import Controller, delay_denial, \
cors_validation, clear_info_cache
cors_validation, set_info_cache, clear_info_cache
from swift.common.storage_policy import POLICIES
from swift.common.swob import HTTPBadRequest, HTTPForbidden, \
HTTPNotFound
@ -85,11 +85,16 @@ class ContainerController(Controller):
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
if not self.account_info(self.account_name, req)[1]:
ai = self.account_info(self.account_name, req)
if not ai[1]:
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
# Don't cache this. It doesn't reflect the state of the
# container, just that the user can't access it.
return aresp
# Don't cache this. The lack of account will be cached, and that
# is sufficient.
return HTTPNotFound(request=req)
part = self.app.container_ring.get_part(
self.account_name, self.container_name)
@ -99,10 +104,18 @@ class ContainerController(Controller):
resp = self.GETorHEAD_base(
req, _('Container'), node_iter, part,
req.swift_entity_path, concurrency)
# Cache this. We just made a request to a storage node and got
# up-to-date information for the container.
resp.headers['X-Backend-Recheck-Container-Existence'] = str(
self.app.recheck_container_existence)
set_info_cache(self.app, req.environ, self.account_name,
self.container_name, resp)
if 'swift.authorize' in req.environ:
req.acl = resp.headers.get('x-container-read')
aresp = req.environ['swift.authorize'](req)
if aresp:
# Don't cache this. It doesn't reflect the state of the
# container, just that the user can't access it.
return aresp
if not req.environ.get('swift_owner', False):
for key in self.app.swift_owner_headers:

9
swift/proxy/server.py
View File

@ -36,7 +36,8 @@ from swift.common.utils import cache_from_env, get_logger, \
from swift.common.constraints import check_utf8, valid_api_version
from swift.proxy.controllers import AccountController, ContainerController, \
ObjectControllerRouter, InfoController
from swift.proxy.controllers.base import get_container_info, NodeIter
from swift.proxy.controllers.base import get_container_info, NodeIter, \
DEFAULT_RECHECK_CONTAINER_EXISTENCE, DEFAULT_RECHECK_ACCOUNT_EXISTENCE
from swift.common.swob import HTTPBadRequest, HTTPForbidden, \
HTTPMethodNotAllowed, HTTPNotFound, HTTPPreconditionFailed, \
HTTPServerError, HTTPException, Request, HTTPServiceUnavailable
@ -106,9 +107,11 @@ class Application(object):
self.error_suppression_limit = \
int(conf.get('error_suppression_limit', 10))
self.recheck_container_existence = \
int(conf.get('recheck_container_existence', 60))
int(conf.get('recheck_container_existence',
DEFAULT_RECHECK_CONTAINER_EXISTENCE))
self.recheck_account_existence = \
int(conf.get('recheck_account_existence', 60))
int(conf.get('recheck_account_existence',
DEFAULT_RECHECK_ACCOUNT_EXISTENCE))
self.allow_account_management = \
config_true_value(conf.get('allow_account_management', 'no'))
self.container_ring = container_ring or Ring(swift_dir,

4
test/functional/tests.py
View File

@ -173,7 +173,9 @@ class TestAccount(Base):
finally:
self.env.account.conn.storage_url = was_url
def testPUT(self):
def testPUTError(self):
if load_constraint('allow_account_management'):
raise SkipTest("Allow account management is enabled")
self.env.account.conn.make_request('PUT')
self.assert_status([403, 405])

7
test/probe/brain.py
View File

@ -142,11 +142,16 @@ class BrainSplitter(object):
"""
put container with next storage policy
"""
policy = next(self.policies)
if policy_index is not None:
policy = POLICIES.get_by_index(int(policy_index))
if not policy:
raise ValueError('Unknown policy with index %s' % policy)
elif not self.policy:
policy = next(self.policies)
else:
policy = self.policy
headers = {'X-Storage-Policy': policy.name}
client.put_container(self.url, self.token, self.container_name,
headers=headers)

25
test/probe/test_container_merge_policy_index.py
View File

@ -20,6 +20,7 @@ import unittest
from nose import SkipTest
from six.moves.urllib.parse import urlparse
from swift.common.manager import Manager
from swift.common.internal_client import InternalClient
from swift.common import utils, direct_client
@ -237,6 +238,14 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
orig_policy_index, node))
def test_reconcile_manifest(self):
info_url = "%s://%s/info" % (urlparse(self.url).scheme,
urlparse(self.url).netloc)
proxy_conn = client.http_connection(info_url)
cluster_info = client.get_capabilities(proxy_conn)
if 'slo' not in cluster_info:
raise SkipTest("SLO not enabled in proxy; "
"can't test manifest reconciliation")
# this test is not only testing a split brain scenario on
# multiple policies with mis-placed objects - it even writes out
# a static large object directly to the storage nodes while the
@ -278,18 +287,18 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
write_part(i)
# write manifest
try:
with self.assertRaises(ClientException) as catcher:
client.put_object(self.url, self.token, self.container_name,
self.object_name,
contents=utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
except ClientException as err:
# so as it works out, you can't really upload a multi-part
# manifest for objects that are currently misplaced - you have to
# wait until they're all available - which is about the same as
# some other failure that causes data to be unavailable to the
# proxy at the time of upload
self.assertEqual(err.http_status, 400)
# so as it works out, you can't really upload a multi-part
# manifest for objects that are currently misplaced - you have to
# wait until they're all available - which is about the same as
# some other failure that causes data to be unavailable to the
# proxy at the time of upload
self.assertEqual(catcher.exception.http_status, 400)
# but what the heck, we'll sneak one in just to see what happens...
direct_manifest_name = self.object_name + '-direct-test'

111
test/probe/test_object_expirer.py
View File

@ -13,17 +13,18 @@
# limitations under the License.
import random
import time
import uuid
import unittest
from nose import SkipTest
from swift.common.internal_client import InternalClient
from swift.common.internal_client import InternalClient, UnexpectedResponse
from swift.common.manager import Manager
from swift.common.utils import Timestamp
from test.probe.common import ReplProbeTest, ENABLED_POLICIES
from test.probe.test_container_merge_policy_index import BrainSplitter
from test.probe.brain import BrainSplitter
from swiftclient import client
@ -31,9 +32,6 @@ from swiftclient import client
class TestObjectExpirer(ReplProbeTest):
def setUp(self):
if len(ENABLED_POLICIES) < 2:
raise SkipTest('Need more than one policy')
self.expirer = Manager(['object-expirer'])
self.expirer.start()
err = self.expirer.stop()
@ -53,6 +51,9 @@ class TestObjectExpirer(ReplProbeTest):
self.object_name)
def test_expirer_object_split_brain(self):
if len(ENABLED_POLICIES) < 2:
raise SkipTest('Need more than one policy')
old_policy = random.choice(ENABLED_POLICIES)
wrong_policy = random.choice([p for p in ENABLED_POLICIES
if p != old_policy])
@ -126,5 +127,105 @@ class TestObjectExpirer(ReplProbeTest):
self.assertTrue(Timestamp(metadata['x-backend-timestamp']) >
create_timestamp)
def test_expirer_object_should_not_be_expired(self):
# Current object-expirer checks the correctness via x-if-delete-at
# header that it can be deleted by expirer. If there are objects
# either which doesn't have x-delete-at header as metadata or which
# has different x-delete-at value from x-if-delete-at value,
# object-expirer's delete will fail as 412 PreconditionFailed.
# However, if some of the objects are in handoff nodes, the expirer
# can put the tombstone with the timestamp as same as x-delete-at and
# the object consistency will be resolved as the newer timestamp will
# be winner (in particular, overwritten case w/o x-delete-at). This
# test asserts such a situation that, at least, the overwriten object
# which have larger timestamp than the original expirered date should
# be safe.
def put_object(headers):
# use internal client to PUT objects so that X-Timestamp in headers
# is effective
headers['Content-Length'] = '0'
path = self.client.make_path(
self.account, self.container_name, self.object_name)
try:
self.client.make_request('PUT', path, headers, (2,))
except UnexpectedResponse as e:
self.fail(
'Expected 201 for PUT object but got %s' % e.resp.status)
obj_brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name, 'object', self.policy)
# T(obj_created) < T(obj_deleted with x-delete-at) < T(obj_recreated)
# < T(expirer_executed)
# Recreated obj should be appeared in any split brain case
obj_brain.put_container()
# T(obj_deleted with x-delete-at)
# object-server accepts req only if X-Delete-At is later than 'now'
# so here, T(obj_created) < T(obj_deleted with x-delete-at)
now = time.time()
delete_at = int(now + 2.0)
recreate_at = delete_at + 1.0
put_object(headers={'X-Delete-At': delete_at,
'X-Timestamp': Timestamp(now).normal})
# some object servers stopped to make a situation that the
# object-expirer can put tombstone in the primary nodes.
obj_brain.stop_primary_half()
# increment the X-Timestamp explicitly
# (will be T(obj_deleted with x-delete-at) < T(obj_recreated))
put_object(headers={'X-Object-Meta-Expired': 'False',
'X-Timestamp': Timestamp(recreate_at).normal})
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# sanity, the newer object is still there
try:
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name)
except UnexpectedResponse as e:
self.fail(
'Expected 200 for HEAD object but got %s' % e.resp.status)
self.assertIn('x-object-meta-expired', metadata)
# some object servers recovered
obj_brain.start_primary_half()
# sleep until after recreated_at
while time.time() <= recreate_at:
time.sleep(0.1)
# Now, expirer runs at the time after obj is recreated
self.expirer.once()
# verify that original object was deleted by expirer
obj_brain.stop_handoff_half()
try:
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,))
except UnexpectedResponse as e:
self.fail(
'Expected 404 for HEAD object but got %s' % e.resp.status)
obj_brain.start_handoff_half()
# and inconsistent state of objects is recovered by replicator
Manager(['object-replicator']).once()
# check if you can get recreated object
try:
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name)
except UnexpectedResponse as e:
self.fail(
'Expected 200 for HEAD object but got %s' % e.resp.status)
self.assertIn('x-object-meta-expired', metadata)
if __name__ == "__main__":
unittest.main()

15
test/unit/common/middleware/test_account_quotas.py
View File

@ -18,9 +18,8 @@ from swift.common.swob import Request, wsgify, HTTPForbidden, \
from swift.common.middleware import account_quotas, copy
from swift.proxy.controllers.base import _get_cache_key, \
headers_to_account_info, get_object_env_key, \
headers_to_object_info
from swift.proxy.controllers.base import get_cache_key, \
headers_to_account_info, headers_to_object_info
class FakeCache(object):
@ -58,16 +57,18 @@ class FakeApp(object):
return aresp(env, start_response)
if env['REQUEST_METHOD'] == "HEAD" and \
env['PATH_INFO'] == '/v1/a/c2/o2':
env_key = get_object_env_key('a', 'c2', 'o2')
env[env_key] = headers_to_object_info(self.headers, 200)
cache_key = get_cache_key('a', 'c2', 'o2')
env.setdefault('swift.infocache', {})[cache_key] = \
headers_to_object_info(self.headers, 200)
start_response('200 OK', self.headers)
elif env['REQUEST_METHOD'] == "HEAD" and \
env['PATH_INFO'] == '/v1/a/c2/o3':
start_response('404 Not Found', [])
else:
# Cache the account_info (same as a real application)
cache_key, env_key = _get_cache_key('a', None)
env[env_key] = headers_to_account_info(self.headers, 200)
cache_key = get_cache_key('a')
env.setdefault('swift.infocache', {})[cache_key] = \
headers_to_account_info(self.headers, 200)
start_response('200 OK', self.headers)
return []

11
test/unit/common/middleware/test_container_sync.py
View File

@ -23,7 +23,7 @@ import mock
from swift.common import swob
from swift.common.middleware import container_sync
from swift.proxy.controllers.base import _get_cache_key
from swift.proxy.controllers.base import get_cache_key
from swift.proxy.controllers.info import InfoController
from test.unit import FakeLogger
@ -205,7 +205,8 @@ cluster_dfw1 = http://dfw1.host/v1/
def test_invalid_sig(self):
req = swob.Request.blank(
'/v1/a/c', headers={'x-container-sync-auth': 'US nonce sig'})
req.environ[_get_cache_key('a', 'c')[1]] = {'sync_key': 'abc'}
infocache = req.environ.setdefault('swift.infocache', {})
infocache[get_cache_key('a', 'c')] = {'sync_key': 'abc'}
resp = req.get_response(self.sync)
self.assertEqual(resp.status, '401 Unauthorized')
self.assertEqual(
@ -224,7 +225,8 @@ cluster_dfw1 = http://dfw1.host/v1/
req = swob.Request.blank('/v1/a/c', headers={
'x-container-sync-auth': 'US nonce ' + sig,
'x-backend-inbound-x-timestamp': ts})
req.environ[_get_cache_key('a', 'c')[1]] = {'sync_key': 'abc'}
infocache = req.environ.setdefault('swift.infocache', {})
infocache[get_cache_key('a', 'c')] = {'sync_key': 'abc'}
resp = req.get_response(self.sync)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body, 'Response to Authorized Request')
@ -238,7 +240,8 @@ cluster_dfw1 = http://dfw1.host/v1/
self.sync.realms_conf.key2('US'), 'abc')
req = swob.Request.blank(
'/v1/a/c', headers={'x-container-sync-auth': 'US nonce ' + sig})
req.environ[_get_cache_key('a', 'c')[1]] = {'sync_key': 'abc'}
infocache = req.environ.setdefault('swift.infocache', {})
infocache[get_cache_key('a', 'c')] = {'sync_key': 'abc'}
resp = req.get_response(self.sync)
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.body, 'Response to Authorized Request')

1
test/unit/common/middleware/test_copy.py
View File

@ -1073,6 +1073,7 @@ class TestServerSideCopyConfiguration(unittest.TestCase):
@patch_policies(with_ec_default=True)
class TestServerSideCopyMiddlewareWithEC(unittest.TestCase):
container_info = {
'status': 200,
'write_acl': None,
'read_acl': None,
'storage_policy': None,

198
test/unit/common/middleware/test_formpost.py
View File

@ -24,6 +24,7 @@ from six import BytesIO
from swift.common.swob import Request, Response
from swift.common.middleware import tempauth, formpost
from swift.common.utils import split_path
from swift.proxy.controllers.base import get_cache_key
class FakeApp(object):
@ -130,8 +131,9 @@ class TestFormPost(unittest.TestCase):
meta[meta_name] = key
_junk, account, _junk, _junk = split_path(path, 2, 4)
req.environ['swift.account/' + account] = self._fake_cache_env(
account, tempurl_keys)
req.environ.setdefault('swift.infocache', {})
req.environ['swift.infocache'][get_cache_key(account)] = \
self._fake_cache_env(account, tempurl_keys)
return req
def _fake_cache_env(self, account, tempurl_keys=()):
@ -221,6 +223,7 @@ class TestFormPost(unittest.TestCase):
'SERVER_NAME': '172.16.83.128',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'swift.infocache': {},
'wsgi.errors': wsgi_errors,
'wsgi.multiprocess': False,
'wsgi.multithread': True,
@ -247,8 +250,8 @@ class TestFormPost(unittest.TestCase):
sig, env, body = self._make_sig_env_body(
'/v1/AUTH_test/container', '', 1024, 10, int(time() - 10), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -351,9 +354,11 @@ class TestFormPost(unittest.TestCase):
'SERVER_NAME': '172.16.83.128',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'swift.account/AUTH_test': self._fake_cache_env(
'AUTH_test', [key]),
'swift.container/AUTH_test/container': {'meta': {}},
'swift.infocache': {
get_cache_key('AUTH_test'): self._fake_cache_env(
'AUTH_test', [key]),
get_cache_key('AUTH_test', 'container'): {
'meta': {}}},
'wsgi.errors': wsgi_errors,
'wsgi.input': wsgi_input,
'wsgi.multiprocess': False,
@ -467,9 +472,11 @@ class TestFormPost(unittest.TestCase):
'SERVER_NAME': '172.16.83.128',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'swift.account/AUTH_test': self._fake_cache_env(
'AUTH_test', [key]),
'swift.container/AUTH_test/container': {'meta': {}},
'swift.infocache': {
get_cache_key('AUTH_test'): self._fake_cache_env(
'AUTH_test', [key]),
get_cache_key('AUTH_test', 'container'): {
'meta': {}}},
'wsgi.errors': wsgi_errors,
'wsgi.input': wsgi_input,
'wsgi.multiprocess': False,
@ -586,9 +593,11 @@ class TestFormPost(unittest.TestCase):
'SERVER_NAME': '172.16.83.128',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'swift.account/AUTH_test': self._fake_cache_env(
'AUTH_test', [key]),
'swift.container/AUTH_test/container': {'meta': {}},
'swift.infocache': {
get_cache_key('AUTH_test'): self._fake_cache_env(
'AUTH_test', [key]),
get_cache_key('AUTH_test', 'container'): {
'meta': {}}},
'wsgi.errors': wsgi_errors,
'wsgi.input': wsgi_input,
'wsgi.multiprocess': False,
@ -701,9 +710,11 @@ class TestFormPost(unittest.TestCase):
'SERVER_NAME': '172.16.83.128',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'swift.account/AUTH_test': self._fake_cache_env(
'AUTH_test', [key]),
'swift.container/AUTH_test/container': {'meta': {}},
'swift.infocache': {
get_cache_key('AUTH_test'): self._fake_cache_env(
'AUTH_test', [key]),
get_cache_key('AUTH_test', 'container'): {
'meta': {}}},
'wsgi.errors': wsgi_errors,
'wsgi.input': wsgi_input,
'wsgi.multiprocess': False,
@ -747,9 +758,10 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', 'http://brim.net', 5, 10,
int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'XX' + b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -783,9 +795,10 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', 'http://brim.net', 5, 10,
int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -814,9 +827,10 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', 'http://brim.net', 1024, 1,
int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -855,9 +869,10 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key)
env['QUERY_STRING'] = 'this=should&not=get&passed'
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(
iter([('201 Created', {}, ''),
('201 Created', {}, '')]),
@ -890,9 +905,10 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', 'http://brim.net', 1024, 10,
int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(iter([('404 Not Found', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -976,9 +992,10 @@ class TestFormPost(unittest.TestCase):
if six.PY3:
wsgi_input = wsgi_input.encode('utf-8')
env['wsgi.input'] = BytesIO(wsgi_input)
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1047,9 +1064,10 @@ class TestFormPost(unittest.TestCase):
if six.PY3:
wsgi_input = wsgi_input.encode('utf-8')
env['wsgi.input'] = BytesIO(wsgi_input)
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1087,9 +1105,10 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', 'http://redirect', 1024, 10,
int(time() + 86400), key, user_agent=False)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1108,9 +1127,10 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', 'http://redirect', 1024, 10,
int(time() + 86400), key, user_agent=False)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
env['HTTP_ORIGIN'] = 'http://localhost:5000'
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created',
@ -1137,9 +1157,10 @@ class TestFormPost(unittest.TestCase):
int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
# Stick it in X-Account-Meta-Temp-URL-Key-2 and make sure we get it
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', ['bert', key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1173,9 +1194,11 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', 'http://redirect', 1024, 10,
int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env('AUTH_test')
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test'))
# Stick it in X-Container-Meta-Temp-URL-Key-2 and ensure we get it
env['swift.container/AUTH_test/container'] = {'meta': meta}
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': meta}
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1199,9 +1222,10 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', 'http://redirect', 1024, 10,
int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1237,9 +1261,10 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', 'http://redirect?one=two', 1024, 10,
int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1275,9 +1300,10 @@ class TestFormPost(unittest.TestCase):
sig, env, body = self._make_sig_env_body(
'/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1312,8 +1338,8 @@ class TestFormPost(unittest.TestCase):
sig, env, body = self._make_sig_env_body(
'/v1/AUTH_test/container', '', 1024, 10, int(time() - 10), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1346,8 +1372,8 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
# Change key to invalidate sig
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key + ' is bogus now'])
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key + ' is bogus now']))
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1379,8 +1405,8 @@ class TestFormPost(unittest.TestCase):
sig, env, body = self._make_sig_env_body(
'/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'XX' + b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1412,8 +1438,8 @@ class TestFormPost(unittest.TestCase):
sig, env, body = self._make_sig_env_body(
'/v2/AUTH_test/container', '', 1024, 10, int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1445,8 +1471,8 @@ class TestFormPost(unittest.TestCase):
sig, env, body = self._make_sig_env_body(
'//AUTH_test/container', '', 1024, 10, int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1478,8 +1504,8 @@ class TestFormPost(unittest.TestCase):
sig, env, body = self._make_sig_env_body(
'/v1//container', '', 1024, 10, int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1511,8 +1537,8 @@ class TestFormPost(unittest.TestCase):
sig, env, body = self._make_sig_env_body(
'/v1/AUTH_tst/container', '', 1024, 10, int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
self.app = FakeApp(iter([
('200 Ok', {'x-account-meta-temp-url-key': 'def'}, ''),
('201 Created', {}, ''),
@ -1546,8 +1572,8 @@ class TestFormPost(unittest.TestCase):
sig, env, body = self._make_sig_env_body(
'/v1/AUTH_test', '', 1024, 10, int(time() + 86400), key)
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1584,8 +1610,8 @@ class TestFormPost(unittest.TestCase):
body[i] = 'badvalue'
break
env['wsgi.input'] = BytesIO(b'\r\n'.join(body))
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1625,9 +1651,10 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key)
wsgi_input = b'\r\n'.join(x_delete_body_part + body)
env['wsgi.input'] = BytesIO(wsgi_input)
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1668,8 +1695,8 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key)
wsgi_input = b'\r\n'.join(x_delete_body_part + body)
env['wsgi.input'] = BytesIO(wsgi_input)
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1703,9 +1730,10 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key)
wsgi_input = b'\r\n'.join(x_delete_body_part + body)
env['wsgi.input'] = BytesIO(wsgi_input)
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.container/AUTH_test/container'] = {'meta': {}}
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
env['swift.infocache'][get_cache_key(
'AUTH_test', 'container')] = {'meta': {}}
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)
@ -1746,8 +1774,8 @@ class TestFormPost(unittest.TestCase):
'/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key)
wsgi_input = b'\r\n'.join(x_delete_body_part + body)
env['wsgi.input'] = BytesIO(wsgi_input)
env['swift.account/AUTH_test'] = self._fake_cache_env(
'AUTH_test', [key])
env['swift.infocache'][get_cache_key('AUTH_test')] = (
self._fake_cache_env('AUTH_test', [key]))
self.app = FakeApp(iter([('201 Created', {}, ''),
('201 Created', {}, '')]))
self.auth = tempauth.filter_factory({})(self.app)

42
test/unit/common/middleware/test_keystoneauth.py
View File

@ -19,7 +19,7 @@ from swift.common.middleware import keystoneauth
from swift.common.swob import Request, Response
from swift.common.http import HTTP_FORBIDDEN
from swift.common.utils import split_path
from swift.proxy.controllers.base import _get_cache_key
from swift.proxy.controllers.base import get_cache_key
from test.unit import FakeLogger
UNKNOWN_ID = keystoneauth.UNKNOWN_ID
@ -251,8 +251,8 @@ class SwiftAuth(unittest.TestCase):
account = get_account_for_tenant(self.test_auth, proj_id)
path = '/v1/' + account
# fake cached account info
_, info_key = _get_cache_key(account, None)
env = {info_key: {'status': 0, 'sysmeta': {}},
info_key = get_cache_key(account)
env = {'swift.infocache': {info_key: {'status': 0, 'sysmeta': {}}},
'keystone.token_info': _fake_token_info(version='3')}
req = Request.blank(path, environ=env, headers=headers)
req.method = 'POST'
@ -280,8 +280,8 @@ class SwiftAuth(unittest.TestCase):
account = get_account_for_tenant(self.test_auth, proj_id)
path = '/v1/' + account
# fake cached account info
_, info_key = _get_cache_key(account, None)
env = {info_key: {'status': 0, 'sysmeta': {}},
info_key = get_cache_key(account)
env = {'swift.infocache': {info_key: {'status': 0, 'sysmeta': {}}},
'keystone.token_info': _fake_token_info(version='3')}
req = Request.blank(path, environ=env, headers=headers)
req.method = 'POST'
@ -301,9 +301,9 @@ class SwiftAuth(unittest.TestCase):
headers = get_identity_headers(tenant_id=proj_id, role='admin')
account = get_account_for_tenant(self.test_auth, proj_id)
path = '/v1/' + account
_, info_key = _get_cache_key(account, None)
info_key = get_cache_key(account)
# v2 token
env = {info_key: {'status': 0, 'sysmeta': {}},
env = {'swift.infocache': {info_key: {'status': 0, 'sysmeta': {}}},
'keystone.token_info': _fake_token_info(version='2')}
req = Request.blank(path, environ=env, headers=headers)
req.method = 'POST'
@ -323,9 +323,9 @@ class SwiftAuth(unittest.TestCase):
role='reselleradmin')
account = get_account_for_tenant(self.test_auth, proj_id)
path = '/v1/' + account
_, info_key = _get_cache_key(account, None)
info_key = get_cache_key(account)
# v2 token
env = {info_key: {'status': 0, 'sysmeta': {}},
env = {'swift.infocache': {info_key: {'status': 0, 'sysmeta': {}}},
'keystone.token_info': _fake_token_info(version='2')}
req = Request.blank(path, environ=env, headers=headers)
req.method = 'POST'
@ -381,8 +381,8 @@ class ServiceTokenFunctionality(unittest.TestCase):
role=user_role,
service_role=service_role)
(version, account, _junk, _junk) = split_path(path, 2, 4, True)
_, info_key = _get_cache_key(account, None)
env = {info_key: {'status': 0, 'sysmeta': {}},
info_key = get_cache_key(account)
env = {'swift.infocache': {info_key: {'status': 0, 'sysmeta': {}}},
'keystone.token_info': _fake_token_info(version='2')}
if environ:
env.update(environ)
@ -595,9 +595,10 @@ class TestAuthorize(BaseTestAuthorize):
if not path:
path = '/v1/%s/c' % account
# fake cached account info
_, info_key = _get_cache_key(account, None)
default_env = {'REMOTE_USER': identity['HTTP_X_TENANT_ID'],
info_key: {'status': 200, 'sysmeta': {}}}
info_key = get_cache_key(account)
default_env = {
'REMOTE_USER': identity['HTTP_X_TENANT_ID'],
'swift.infocache': {info_key: {'status': 200, 'sysmeta': {}}}}
default_env.update(identity)
if env:
default_env.update(env)
@ -984,9 +985,9 @@ class TestAuthorize(BaseTestAuthorize):
def test_get_project_domain_id(self):
sysmeta = {}
info = {'sysmeta': sysmeta}
_, info_key = _get_cache_key('AUTH_1234', None)
info_key = get_cache_key('AUTH_1234')
env = {'PATH_INFO': '/v1/AUTH_1234',
info_key: info}
'swift.infocache': {info_key: info}}
# account does not exist
info['status'] = 404
@ -1028,8 +1029,9 @@ class TestIsNameAllowedInACL(BaseTestAuthorize):
# pretend account exists
info = {'status': 200, 'sysmeta': sysmeta}
_, info_key = _get_cache_key(account, None)
req = Request.blank(path, environ={info_key: info})
info_key = get_cache_key(account)
req = Request.blank(path,
environ={'swift.infocache': {info_key: info}})
if scoped == 'account':
project_name = 'account_name'
@ -1214,8 +1216,8 @@ class TestSetProjectDomain(BaseTestAuthorize):
if sysmeta_project_domain_id:
sysmeta['project-domain-id'] = sysmeta_project_domain_id
info = {'status': status, 'sysmeta': sysmeta}
_, info_key = _get_cache_key(account, None)
env = {info_key: info}
info_key = get_cache_key(account)
env = {'swift.infocache': {info_key: info}}
# create fake env identity
env_id = self._get_env_id(tenant_id=req_project_id,

10
test/unit/common/middleware/test_quotas.py
View File

@ -245,8 +245,9 @@ class ContainerQuotaCopyingTestCases(unittest.TestCase):
'status': 200, 'object_count': 1}
req = Request.blank('/v1/a/c2/o2',
environ={'REQUEST_METHOD': 'COPY',
'swift.container/a/c': a_c_cache,
'swift.container/a2/c': a2_c_cache},
'swift.infocache': {
'container/a/c': a_c_cache,
'container/a2/c': a2_c_cache}},
headers={'Destination': '/c/o',
'Destination-Account': 'a2'})
res = req.get_response(self.copy_filter)
@ -261,8 +262,9 @@ class ContainerQuotaCopyingTestCases(unittest.TestCase):
'status': 200, 'object_count': 1}
req = Request.blank('/v1/a2/c/o',
environ={'REQUEST_METHOD': 'PUT',
'swift.container/a/c': a_c_cache,
'swift.container/a2/c': a2_c_cache},
'swift.infocache': {
'container/a/c': a_c_cache,
'container/a2/c': a2_c_cache}},
headers={'X-Copy-From': '/c2/o2',
'X-Copy-From-Account': 'a'})
res = req.get_response(self.copy_filter)

14
test/unit/common/middleware/test_ratelimit.py
View File

@ -21,7 +21,7 @@ from contextlib import contextmanager
from test.unit import FakeLogger
from swift.common.middleware import ratelimit
from swift.proxy.controllers.base import get_container_memcache_key, \
from swift.proxy.controllers.base import get_cache_key, \
headers_to_container_info
from swift.common.memcached import MemcacheConnectionError
from swift.common.swob import Request
@ -185,7 +185,7 @@ class TestRateLimit(unittest.TestCase):
conf_dict = {'account_ratelimit': current_rate,
'container_ratelimit_3': 200}
fake_memcache = FakeMemcache()
fake_memcache.store[get_container_memcache_key('a', 'c')] = \
fake_memcache.store[get_cache_key('a', 'c')] = \
{'object_count': '5'}
the_app = ratelimit.filter_factory(conf_dict)(FakeApp())
the_app.memcache_client = fake_memcache
@ -229,7 +229,7 @@ class TestRateLimit(unittest.TestCase):
conf_dict = {'account_ratelimit': current_rate,
'container_ratelimit_3': 200}
fake_memcache = FakeMemcache()
fake_memcache.store[get_container_memcache_key('a', 'c')] = \
fake_memcache.store[get_cache_key('a', 'c')] = \
{'container_size': 5}
the_app = ratelimit.filter_factory(conf_dict)(FakeApp())
the_app.memcache_client = fake_memcache
@ -431,8 +431,8 @@ class TestRateLimit(unittest.TestCase):
req.method = 'PUT'
req.environ['swift.cache'] = FakeMemcache()
req.environ['swift.cache'].set(
get_container_memcache_key('a', 'c'),
{'container_size': 1})
get_cache_key('a', 'c'),
{'object_count': 1})
time_override = [0, 0, 0, 0, None]
# simulates 4 requests coming in at same time, then sleeping
@ -465,8 +465,8 @@ class TestRateLimit(unittest.TestCase):
req.method = 'GET'
req.environ['swift.cache'] = FakeMemcache()
req.environ['swift.cache'].set(
get_container_memcache_key('a', 'c'),
{'container_size': 1})
get_cache_key('a', 'c'),
{'object_count': 1})
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}):

12
test/unit/common/middleware/test_tempurl.py
View File

@ -96,7 +96,8 @@ class TestTempURL(unittest.TestCase):
if key:
meta[meta_name] = key
environ['swift.account/' + account] = {
ic = environ.setdefault('swift.infocache', {})
ic['account/' + account] = {
'status': 204,
'container_count': '0',
'total_object_count': '0',
@ -108,8 +109,8 @@ class TestTempURL(unittest.TestCase):
meta_name = 'Temp-URL-key' + (("-%d" % (i + 1) if i else ""))
meta[meta_name] = key
container_cache_key = 'swift.container/' + account + '/c'
environ.setdefault(container_cache_key, {'meta': meta})
container_cache_key = 'container/' + account + '/c'
ic.setdefault(container_cache_key, {'meta': meta})
def test_passthrough(self):
resp = self._make_request('/v1/a/c/o').get_response(self.tempurl)
@ -159,7 +160,8 @@ class TestTempURL(unittest.TestCase):
self.assert_valid_sig(expires, path, [key1, key2], sig)
def test_get_valid_container_keys(self):
environ = {}
ic = {}
environ = {'swift.infocache': ic}
# Add two static container keys
container_keys = ['me', 'other']
meta = {}
@ -167,7 +169,7 @@ class TestTempURL(unittest.TestCase):
meta_name = 'Temp-URL-key' + (("-%d" % (idx + 1) if idx else ""))
if key:
meta[meta_name] = key
environ['swift.container/a/c'] = {'meta': meta}
ic['container/a/c'] = {'meta': meta}
method = 'GET'
expires = int(time() + 86400)

29
test/unit/common/test_db_replicator.py
View File

@ -73,7 +73,7 @@ class FakeRingWithSingleNode(object):
class Ring(object):
devs = [dict(
id=1, weight=10.0, zone=1, ip='1.1.1.1', port=6200, device='sdb',
meta='', replication_ip='1.1.1.1', replication_port=6200
meta='', replication_ip='1.1.1.1', replication_port=6200, region=1
)]
def __init__(self, path, reload_time=15, ring_name=None):
@ -633,6 +633,9 @@ class TestDBReplicator(unittest.TestCase):
def test_replicate_object_delete_because_not_shouldbehere(self):
replicator = TestReplicator({})
replicator.ring = FakeRingWithNodes().Ring('path')
replicator.brokerclass = FakeAccountBroker
replicator._repl_to_node = lambda *args: True
replicator.delete_db = self.stub_delete_db
replicator._replicate_object('0', '/path/to/file', 'node_id')
self.assertEqual(['/path/to/file'], self.delete_db_calls)
@ -669,6 +672,30 @@ class TestDBReplicator(unittest.TestCase):
[(('Found /path/to/file for /a%20c%20t/c%20o%20n when it should '
'be on partition 0; will replicate out and remove.',), {})])
def test_replicate_container_out_of_place_no_node(self):
replicator = TestReplicator({}, logger=unit.FakeLogger())
replicator.ring = FakeRingWithSingleNode().Ring('path')
replicator._repl_to_node = lambda *args: True
replicator.delete_db = self.stub_delete_db
# Correct node_id, wrong part
part = replicator.ring.get_part(
TEST_ACCOUNT_NAME, TEST_CONTAINER_NAME) + 1
node_id = replicator.ring.get_part_nodes(part)[0]['id']
replicator._replicate_object(str(part), '/path/to/file', node_id)
self.assertEqual(['/path/to/file'], self.delete_db_calls)
self.delete_db_calls = []
# No nodes this time
replicator.ring.get_part_nodes = lambda *args: []
replicator.delete_db = self.stub_delete_db
# Correct node_id, wrong part
part = replicator.ring.get_part(
TEST_ACCOUNT_NAME, TEST_CONTAINER_NAME) + 1
replicator._replicate_object(str(part), '/path/to/file', node_id)
self.assertEqual([], self.delete_db_calls)
def test_replicate_object_different_region(self):
db_replicator.ring = FakeRingWithNodes()
replicator = TestReplicator({})

146
test/unit/common/test_utils.py
View File

@ -2601,6 +2601,19 @@ cluster_dfw1 = http://dfw1.host/v1/
try:
fallocate = utils.FallocateWrapper(noop=True)
utils.os.fstatvfs = fstatvfs
# Make sure setting noop, which disables fallocate, also stops the
# fallocate_reserve check.
# Set the fallocate_reserve to 99% and request an object that is
# about 50% the size. With fallocate_reserve off this will succeed.
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('99%')
self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(500)), 0)
# Setting noop to False after the constructor allows us to use
# a noop fallocate syscall and still test fallocate_reserve.
fallocate.noop = False
# Want 1023 reserved, have 1024 * 1 free, so succeeds
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('1023')
@ -2618,67 +2631,68 @@ cluster_dfw1 = http://dfw1.host/v1/
utils.config_fallocate_value('1024')
StatVFS.f_frsize = 1024
StatVFS.f_bavail = 1
exc = None
try:
with self.assertRaises(OSError) as catcher:
fallocate(0, 1, 0, ctypes.c_uint64(0))
except OSError as err:
exc = err
self.assertEqual(str(exc),
'[Errno 28] FALLOCATE_RESERVE fail 1024 <= 1024')
self.assertEqual(err.errno, errno.ENOSPC)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
% errno.ENOSPC)
self.assertEqual(catcher.exception.errno, errno.ENOSPC)
# Want 1024 reserved, have 512 * 2 free, so fails
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('1024')
StatVFS.f_frsize = 512
StatVFS.f_bavail = 2
exc = None
try:
with self.assertRaises(OSError) as catcher:
fallocate(0, 1, 0, ctypes.c_uint64(0))
except OSError as err:
exc = err
self.assertEqual(str(exc),
'[Errno 28] FALLOCATE_RESERVE fail 1024 <= 1024')
self.assertEqual(err.errno, errno.ENOSPC)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
% errno.ENOSPC)
self.assertEqual(catcher.exception.errno, errno.ENOSPC)
# Want 2048 reserved, have 1024 * 1 free, so fails
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('2048')
StatVFS.f_frsize = 1024
StatVFS.f_bavail = 1
exc = None
try:
with self.assertRaises(OSError) as catcher:
fallocate(0, 1, 0, ctypes.c_uint64(0))
except OSError as err:
exc = err
self.assertEqual(str(exc),
'[Errno 28] FALLOCATE_RESERVE fail 1024 <= 2048')
self.assertEqual(err.errno, errno.ENOSPC)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 1024 <= 2048'
% errno.ENOSPC)
self.assertEqual(catcher.exception.errno, errno.ENOSPC)
# Want 2048 reserved, have 512 * 2 free, so fails
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('2048')
StatVFS.f_frsize = 512
StatVFS.f_bavail = 2
exc = None
try:
with self.assertRaises(OSError) as catcher:
fallocate(0, 1, 0, ctypes.c_uint64(0))
except OSError as err:
exc = err
self.assertEqual(str(exc),
'[Errno 28] FALLOCATE_RESERVE fail 1024 <= 2048')
self.assertEqual(err.errno, errno.ENOSPC)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 1024 <= 2048'
% errno.ENOSPC)
self.assertEqual(catcher.exception.errno, errno.ENOSPC)
# Want 1023 reserved, have 1024 * 1 free, but file size is 1, so
# fails
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('1023')
StatVFS.f_frsize = 1024
StatVFS.f_bavail = 1
exc = None
try:
with self.assertRaises(OSError) as catcher:
fallocate(0, 1, 0, ctypes.c_uint64(1))
except OSError as err:
exc = err
self.assertEqual(str(exc),
'[Errno 28] FALLOCATE_RESERVE fail 1023 <= 1023')
self.assertEqual(err.errno, errno.ENOSPC)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 1023 <= 1023'
% errno.ENOSPC)
self.assertEqual(catcher.exception.errno, errno.ENOSPC)
# Want 1022 reserved, have 1024 * 1 free, and file size is 1, so
# succeeds
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
@ -2686,6 +2700,7 @@ cluster_dfw1 = http://dfw1.host/v1/
StatVFS.f_frsize = 1024
StatVFS.f_bavail = 1
self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(1)), 0)
# Want 1% reserved, have 100 bytes * 2/100 free, and file size is
# 99, so succeeds
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
@ -2694,6 +2709,7 @@ cluster_dfw1 = http://dfw1.host/v1/
StatVFS.f_bavail = 2
StatVFS.f_blocks = 100
self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(99)), 0)
# Want 2% reserved, have 50 bytes * 2/50 free, and file size is 49,
# so succeeds
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
@ -2702,6 +2718,7 @@ cluster_dfw1 = http://dfw1.host/v1/
StatVFS.f_bavail = 2
StatVFS.f_blocks = 50
self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(49)), 0)
# Want 100% reserved, have 100 * 100/100 free, and file size is 0,
# so fails.
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
@ -2709,15 +2726,14 @@ cluster_dfw1 = http://dfw1.host/v1/
StatVFS.f_frsize = 100
StatVFS.f_bavail = 100
StatVFS.f_blocks = 100
exc = None
try:
with self.assertRaises(OSError) as catcher:
fallocate(0, 1, 0, ctypes.c_uint64(0))
except OSError as err:
exc = err
self.assertEqual(str(exc),
'[Errno 28] FALLOCATE_RESERVE fail 100.0 <= '
'100.0')
self.assertEqual(err.errno, errno.ENOSPC)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 100.0 <= 100.0'
% errno.ENOSPC)
self.assertEqual(catcher.exception.errno, errno.ENOSPC)
# Want 1% reserved, have 100 * 2/100 free, and file size is 101,
# so fails.
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
@ -2725,29 +2741,28 @@ cluster_dfw1 = http://dfw1.host/v1/
StatVFS.f_frsize = 100
StatVFS.f_bavail = 2
StatVFS.f_blocks = 100
exc = None
try:
with self.assertRaises(OSError) as catcher:
fallocate(0, 1, 0, ctypes.c_uint64(101))
except OSError as err:
exc = err
self.assertEqual(str(exc),
'[Errno 28] FALLOCATE_RESERVE fail 0.99 <= 1.0')
self.assertEqual(err.errno, errno.ENOSPC)
# Want 98% reserved, have 100 bytes * 99/100 free, and file size
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 0.99 <= 1.0'
% errno.ENOSPC)
self.assertEqual(catcher.exception.errno, errno.ENOSPC)
# is 100, so fails
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('98%')
StatVFS.f_frsize = 100
StatVFS.f_bavail = 99
StatVFS.f_blocks = 100
exc = None
try:
with self.assertRaises(OSError) as catcher:
fallocate(0, 1, 0, ctypes.c_uint64(100))
except OSError as err:
exc = err
self.assertEqual(str(exc),
'[Errno 28] FALLOCATE_RESERVE fail 98.0 <= 98.0')
self.assertEqual(err.errno, errno.ENOSPC)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 98.0 <= 98.0'
% errno.ENOSPC)
self.assertEqual(catcher.exception.errno, errno.ENOSPC)
# Want 2% reserved, have 1000 bytes * 21/1000 free, and file size
# is 999, so succeeds.
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
@ -2756,6 +2771,7 @@ cluster_dfw1 = http://dfw1.host/v1/
StatVFS.f_bavail = 21
StatVFS.f_blocks = 1000
self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(999)), 0)
# Want 2% resereved, have 1000 bytes * 21/1000 free, and file size
# is 1000, so fails.
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
@ -2763,14 +2779,14 @@ cluster_dfw1 = http://dfw1.host/v1/
StatVFS.f_frsize = 1000
StatVFS.f_bavail = 21
StatVFS.f_blocks = 1000
exc = None
try:
with self.assertRaises(OSError) as catcher:
fallocate(0, 1, 0, ctypes.c_uint64(1000))
except OSError as err:
exc = err
self.assertEqual(str(exc),
'[Errno 28] FALLOCATE_RESERVE fail 2.0 <= 2.0')
self.assertEqual(err.errno, errno.ENOSPC)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 2.0 <= 2.0'
% errno.ENOSPC)
self.assertEqual(catcher.exception.errno, errno.ENOSPC)
finally:
utils.FALLOCATE_RESERVE = orig_FALLOCATE_RESERVE
utils.os.fstatvfs = orig_fstatvfs

5
test/unit/common/test_wsgi.py
View File

@ -839,6 +839,11 @@ class TestWSGI(unittest.TestCase):
self.assertTrue('HTTP_REFERER' in newenv)
self.assertEqual(newenv['HTTP_REFERER'], 'http://blah.example.com')
def test_make_env_keeps_infocache(self):
oldenv = {'swift.infocache': {}}
newenv = wsgi.make_env(oldenv)
self.assertIs(newenv.get('swift.infocache'), oldenv['swift.infocache'])
class TestServersPerPortStrategy(unittest.TestCase):
def setUp(self):

4
test/unit/obj/test_expirer.py
View File

@ -675,6 +675,8 @@ class TestObjectExpirer(TestCase):
ts = '1234'
x.delete_actual_object('/path/to/object', ts)
self.assertEqual(got_env[0]['HTTP_X_IF_DELETE_AT'], ts)
self.assertEqual(got_env[0]['HTTP_X_TIMESTAMP'],
got_env[0]['HTTP_X_IF_DELETE_AT'])
def test_delete_actual_object_nourlquoting(self):
# delete_actual_object should not do its own url quoting because
@ -692,6 +694,8 @@ class TestObjectExpirer(TestCase):
ts = '1234'
x.delete_actual_object('/path/to/object name', ts)
self.assertEqual(got_env[0]['HTTP_X_IF_DELETE_AT'], ts)
self.assertEqual(got_env[0]['HTTP_X_TIMESTAMP'],
got_env[0]['HTTP_X_IF_DELETE_AT'])
self.assertEqual(got_env[0]['PATH_INFO'], '/v1/path/to/object name')
def test_delete_actual_object_raises_404(self):

39
test/unit/obj/test_reconstructor.py
View File

@ -823,8 +823,43 @@ class TestGlobalSetupObjectReconstructor(unittest.TestCase):
self.assertFalse(os.path.exists(pol_1_part_1_path))
warnings = self.reconstructor.logger.get_lines_for_level('warning')
self.assertEqual(1, len(warnings))
self.assertTrue('Unexpected entity in data dir:' in warnings[0],
'Warning not found in %s' % warnings)
self.assertIn('Unexpected entity in data dir:', warnings[0])
def test_ignores_status_file(self):
# Following fd86d5a, the auditor will leave status files on each device
# until an audit can complete. The reconstructor should ignore these
@contextmanager
def status_files(*auditor_types):
status_paths = [os.path.join(self.objects_1,
'auditor_status_%s.json' % typ)
for typ in auditor_types]
for status_path in status_paths:
self.assertFalse(os.path.exists(status_path)) # sanity check
with open(status_path, 'w'):
pass
self.assertTrue(os.path.isfile(status_path)) # sanity check
try:
yield status_paths
finally:
for status_path in status_paths:
try:
os.unlink(status_path)
except OSError as e:
if e.errno != 2:
raise
# since our collect_parts job is a generator, that yields directly
# into build_jobs and then spawns it's safe to do the remove_files
# without making reconstructor startup slow
with status_files('ALL', 'ZBF') as status_paths:
self.reconstructor._reset_stats()
for part_info in self.reconstructor.collect_parts():
self.assertNotIn(part_info['part_path'], status_paths)
warnings = self.reconstructor.logger.get_lines_for_level('warning')
self.assertEqual(0, len(warnings))
for status_path in status_paths:
self.assertTrue(os.path.exists(status_path))
def _make_fake_ssync(self, ssync_calls):
class _fake_ssync(object):

42
test/unit/proxy/controllers/test_account.py
View File

@ -25,6 +25,7 @@ from test.unit import fake_http_connect, FakeRing, FakeMemcache
from swift.common.storage_policy import StoragePolicy
from swift.common.request_helpers import get_sys_meta_prefix
import swift.proxy.controllers.base
from swift.proxy.controllers.base import get_account_info
from test.unit import patch_policies
@ -68,9 +69,10 @@ class TestAccountController(unittest.TestCase):
req = Request.blank('/v1/AUTH_bob', {'PATH_INFO': '/v1/AUTH_bob'})
resp = controller.HEAD(req)
self.assertEqual(2, resp.status_int // 100)
self.assertTrue('swift.account/AUTH_bob' in resp.environ)
self.assertEqual(headers_to_account_info(resp.headers),
resp.environ['swift.account/AUTH_bob'])
self.assertIn('account/AUTH_bob', resp.environ['swift.infocache'])
self.assertEqual(
headers_to_account_info(resp.headers),
resp.environ['swift.infocache']['account/AUTH_bob'])
def test_swift_owner(self):
owner_headers = {
@ -225,13 +227,20 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(1, len(resp.headers)) # we always get Content-Type
self.assertEqual(2, len(resp.environ))
def test_memcache_key_impossible_cases(self):
def test_cache_key_impossible_cases(self):
# For test coverage: verify that defensive coding does defend, in cases
# that shouldn't arise naturally
self.assertRaises(
ValueError,
lambda: swift.proxy.controllers.base.get_container_memcache_key(
'/a', None))
with self.assertRaises(ValueError):
# Container needs account
swift.proxy.controllers.base.get_cache_key(None, 'c')
with self.assertRaises(ValueError):
# Object needs account
swift.proxy.controllers.base.get_cache_key(None, 'c', 'o')
with self.assertRaises(ValueError):
# Object needs container
swift.proxy.controllers.base.get_cache_key('a', None, 'o')
def test_stripping_swift_admin_headers(self):
# Verify that a GET/HEAD which receives privileged headers from the
@ -376,5 +385,22 @@ class TestAccountController4Replicas(TestAccountController):
self._assert_responses('POST', POST_TEST_CASES)
@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())])
class TestGetAccountInfo(unittest.TestCase):
def setUp(self):
self.app = proxy_server.Application(
None, FakeMemcache(),
account_ring=FakeRing(), container_ring=FakeRing())
def test_get_deleted_account_410(self):
resp_headers = {'x-account-status': 'deleted'}
req = Request.blank('/v1/a')
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(404, headers=resp_headers)):
info = get_account_info(req.environ, self.app)
self.assertEqual(410, info.get('status'))
if __name__ == '__main__':
unittest.main()

290
test/unit/proxy/controllers/test_base.py
View File

@ -19,16 +19,14 @@ import unittest
from mock import patch
from swift.proxy.controllers.base import headers_to_container_info, \
headers_to_account_info, headers_to_object_info, get_container_info, \
get_container_memcache_key, get_account_info, get_account_memcache_key, \
get_object_env_key, get_info, get_object_info, \
Controller, GetOrHeadHandler, _set_info_cache, _set_object_info_cache, \
bytes_to_skip
get_cache_key, get_account_info, get_info, get_object_info, \
Controller, GetOrHeadHandler, bytes_to_skip
from swift.common.swob import Request, HTTPException, RESPONSE_REASONS
from swift.common import exceptions
from swift.common.utils import split_path
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.http import is_success
from swift.common.storage_policy import StoragePolicy, POLICIES
from swift.common.storage_policy import StoragePolicy
from test.unit import fake_http_connect, FakeRing, FakeMemcache
from swift.proxy import server as proxy_server
from swift.common.request_helpers import (
@ -122,23 +120,14 @@ class FakeApp(object):
def __init__(self, response_factory=None, statuses=None):
self.responses = response_factory or \
DynamicResponseFactory(*statuses or [])
self.sources = []
self.captured_envs = []
def __call__(self, environ, start_response):
self.sources.append(environ.get('swift.source'))
self.captured_envs.append(environ)
response = self.responses.get_response(environ)
reason = RESPONSE_REASONS[response.status_int][0]
start_response('%d %s' % (response.status_int, reason),
[(k, v) for k, v in response.headers.items()])
# It's a bit strange, but the get_info cache stuff relies on the
# app setting some keys in the environment as it makes requests
# (in particular GETorHEAD_base) - so our fake does the same
_set_info_cache(self, environ, response.account,
response.container, response)
if response.obj:
_set_object_info_cache(self, environ, response.account,
response.container, response.obj,
response)
return iter(response.body)
@ -160,87 +149,6 @@ class TestFuncs(unittest.TestCase):
account_ring=FakeRing(),
container_ring=FakeRing())
def test_GETorHEAD_base(self):
base = Controller(self.app)
req = Request.blank('/v1/a/c/o/with/slashes')
ring = FakeRing()
nodes = list(ring.get_part_nodes(0)) + list(ring.get_more_nodes(0))
with patch('swift.proxy.controllers.base.'
'http_connect', fake_http_connect(200)):
resp = base.GETorHEAD_base(req, 'object', iter(nodes), 'part',
'/a/c/o/with/slashes')
self.assertTrue('swift.object/a/c/o/with/slashes' in resp.environ)
self.assertEqual(
resp.environ['swift.object/a/c/o/with/slashes']['status'], 200)
req = Request.blank('/v1/a/c/o')
with patch('swift.proxy.controllers.base.'
'http_connect', fake_http_connect(200)):
resp = base.GETorHEAD_base(req, 'object', iter(nodes), 'part',
'/a/c/o')
self.assertTrue('swift.object/a/c/o' in resp.environ)
self.assertEqual(resp.environ['swift.object/a/c/o']['status'], 200)
req = Request.blank('/v1/a/c')
with patch('swift.proxy.controllers.base.'
'http_connect', fake_http_connect(200)):
resp = base.GETorHEAD_base(req, 'container', iter(nodes), 'part',
'/a/c')
self.assertTrue('swift.container/a/c' in resp.environ)
self.assertEqual(resp.environ['swift.container/a/c']['status'], 200)
req = Request.blank('/v1/a')
with patch('swift.proxy.controllers.base.'
'http_connect', fake_http_connect(200)):
resp = base.GETorHEAD_base(req, 'account', iter(nodes), 'part',
'/a')
self.assertTrue('swift.account/a' in resp.environ)
self.assertEqual(resp.environ['swift.account/a']['status'], 200)
# Run the above tests again, but this time with concurrent_reads
# turned on
policy = next(iter(POLICIES))
concurrent_get_threads = policy.object_ring.replica_count
for concurrency_timeout in (0, 2):
self.app.concurrency_timeout = concurrency_timeout
req = Request.blank('/v1/a/c/o/with/slashes')
# NOTE: We are using slow_connect of fake_http_connect as using
# a concurrency of 0 when mocking the connection is a little too
# fast for eventlet. Network i/o will make this fine, but mocking
# it seems is too instantaneous.
with patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, slow_connect=True)):
resp = base.GETorHEAD_base(
req, 'object', iter(nodes), 'part', '/a/c/o/with/slashes',
concurrency=concurrent_get_threads)
self.assertTrue('swift.object/a/c/o/with/slashes' in resp.environ)
self.assertEqual(
resp.environ['swift.object/a/c/o/with/slashes']['status'], 200)
req = Request.blank('/v1/a/c/o')
with patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, slow_connect=True)):
resp = base.GETorHEAD_base(
req, 'object', iter(nodes), 'part', '/a/c/o',
concurrency=concurrent_get_threads)
self.assertTrue('swift.object/a/c/o' in resp.environ)
self.assertEqual(resp.environ['swift.object/a/c/o']['status'], 200)
req = Request.blank('/v1/a/c')
with patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, slow_connect=True)):
resp = base.GETorHEAD_base(
req, 'container', iter(nodes), 'part', '/a/c',
concurrency=concurrent_get_threads)
self.assertTrue('swift.container/a/c' in resp.environ)
self.assertEqual(resp.environ['swift.container/a/c']['status'],
200)
req = Request.blank('/v1/a')
with patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, slow_connect=True)):
resp = base.GETorHEAD_base(
req, 'account', iter(nodes), 'part', '/a',
concurrency=concurrent_get_threads)
self.assertTrue('swift.account/a' in resp.environ)
self.assertEqual(resp.environ['swift.account/a']['status'], 200)
def test_get_info(self):
app = FakeApp()
# Do a non cached call to account
@ -250,36 +158,44 @@ class TestFuncs(unittest.TestCase):
self.assertEqual(info_a['status'], 200)
self.assertEqual(info_a['bytes'], 6666)
self.assertEqual(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEqual(env.get('swift.account/a'), info_a)
# Make sure the app was called
self.assertEqual(app.responses.stats['account'], 1)
# Make sure the return value matches get_account_info
account_info = get_account_info({'PATH_INFO': '/v1/a'}, app)
self.assertEqual(info_a, account_info)
# Do an env cached call to account
app.responses.stats['account'] = 0
app.responses.stats['container'] = 0
info_a = get_info(app, env, 'a')
# Check that you got proper info
self.assertEqual(info_a['status'], 200)
self.assertEqual(info_a['bytes'], 6666)
self.assertEqual(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEqual(env.get('swift.account/a'), info_a)
# Make sure the app was NOT called AGAIN
self.assertEqual(app.responses.stats['account'], 1)
self.assertEqual(app.responses.stats['account'], 0)
# This time do env cached call to account and non cached to container
app.responses.stats['account'] = 0
app.responses.stats['container'] = 0
info_c = get_info(app, env, 'a', 'c')
# Check that you got proper info
self.assertEqual(info_c['status'], 200)
self.assertEqual(info_c['bytes'], 6666)
self.assertEqual(info_c['object_count'], 1000)
# Make sure the env cache is set
self.assertEqual(env.get('swift.account/a'), info_a)
self.assertEqual(env.get('swift.container/a/c'), info_c)
# Make sure the app was called for container
# Make sure the app was called for container but not account
self.assertEqual(app.responses.stats['account'], 0)
self.assertEqual(app.responses.stats['container'], 1)
# This time do a non cached call to account than non cached to
# This time do a non-cached call to account then non-cached to
# container
app.responses.stats['account'] = 0
app.responses.stats['container'] = 0
app = FakeApp()
env = {} # abandon previous call to env
info_c = get_info(app, env, 'a', 'c')
@ -287,81 +203,38 @@ class TestFuncs(unittest.TestCase):
self.assertEqual(info_c['status'], 200)
self.assertEqual(info_c['bytes'], 6666)
self.assertEqual(info_c['object_count'], 1000)
# Make sure the env cache is set
self.assertEqual(env.get('swift.account/a'), info_a)
self.assertEqual(env.get('swift.container/a/c'), info_c)
# check app calls both account and container
self.assertEqual(app.responses.stats['account'], 1)
self.assertEqual(app.responses.stats['container'], 1)
# This time do an env cached call to container while account is not
# This time do an env-cached call to container while account is not
# cached
del(env['swift.account/a'])
app.responses.stats['account'] = 0
app.responses.stats['container'] = 0
info_c = get_info(app, env, 'a', 'c')
# Check that you got proper info
self.assertEqual(info_a['status'], 200)
self.assertEqual(info_c['bytes'], 6666)
self.assertEqual(info_c['object_count'], 1000)
# Make sure the env cache is set and account still not cached
self.assertEqual(env.get('swift.container/a/c'), info_c)
# no additional calls were made
self.assertEqual(app.responses.stats['account'], 1)
self.assertEqual(app.responses.stats['container'], 1)
# Do a non cached call to account not found with ret_not_found
app = FakeApp(statuses=(404,))
env = {}
info_a = get_info(app, env, 'a', ret_not_found=True)
# Check that you got proper info
self.assertEqual(info_a['status'], 404)
self.assertEqual(info_a['bytes'], None)
self.assertEqual(info_a['total_object_count'], None)
# Make sure the env cache is set
self.assertEqual(env.get('swift.account/a'), info_a)
# and account was called
self.assertEqual(app.responses.stats['account'], 1)
# Do a cached call to account not found with ret_not_found
info_a = get_info(app, env, 'a', ret_not_found=True)
# Check that you got proper info
self.assertEqual(info_a['status'], 404)
self.assertEqual(info_a['bytes'], None)
self.assertEqual(info_a['total_object_count'], None)
# Make sure the env cache is set
self.assertEqual(env.get('swift.account/a'), info_a)
# add account was NOT called AGAIN
self.assertEqual(app.responses.stats['account'], 1)
# Do a non cached call to account not found without ret_not_found
app = FakeApp(statuses=(404,))
env = {}
info_a = get_info(app, env, 'a')
# Check that you got proper info
self.assertEqual(info_a, None)
self.assertEqual(env['swift.account/a']['status'], 404)
# and account was called
self.assertEqual(app.responses.stats['account'], 1)
# Do a cached call to account not found without ret_not_found
info_a = get_info(None, env, 'a')
# Check that you got proper info
self.assertEqual(info_a, None)
self.assertEqual(env['swift.account/a']['status'], 404)
# add account was NOT called AGAIN
self.assertEqual(app.responses.stats['account'], 1)
self.assertEqual(app.responses.stats['account'], 0)
self.assertEqual(app.responses.stats['container'], 0)
def test_get_container_info_swift_source(self):
app = FakeApp()
req = Request.blank("/v1/a/c", environ={'swift.cache': FakeCache()})
get_container_info(req.environ, app, swift_source='MC')
self.assertEqual(app.sources, ['GET_INFO', 'MC'])
self.assertEqual([e['swift.source'] for e in app.captured_envs],
['MC', 'MC'])
def test_get_object_info_swift_source(self):
app = FakeApp()
req = Request.blank("/v1/a/c/o",
environ={'swift.cache': FakeCache()})
get_object_info(req.environ, app, swift_source='LU')
self.assertEqual(app.sources, ['LU'])
self.assertEqual([e['swift.source'] for e in app.captured_envs],
['LU'])
def test_get_container_info_no_cache(self):
req = Request.blank("/v1/AUTH_account/cont",
@ -379,7 +252,7 @@ class TestFuncs(unittest.TestCase):
self.assertEqual(info['status'], 0)
def test_get_container_info_no_auto_account(self):
responses = DynamicResponseFactory(404, 200)
responses = DynamicResponseFactory(200)
app = FakeApp(responses)
req = Request.blank("/v1/.system_account/cont")
info = get_container_info(req.environ, app)
@ -401,11 +274,11 @@ class TestFuncs(unittest.TestCase):
self.assertEqual(resp['versions'], "\xe1\xbd\x8a\x39")
def test_get_container_info_env(self):
cache_key = get_container_memcache_key("account", "cont")
env_key = 'swift.%s' % cache_key
req = Request.blank("/v1/account/cont",
environ={env_key: {'bytes': 3867},
'swift.cache': FakeCache({})})
cache_key = get_cache_key("account", "cont")
req = Request.blank(
"/v1/account/cont",
environ={'swift.infocache': {cache_key: {'bytes': 3867}},
'swift.cache': FakeCache({})})
resp = get_container_info(req.environ, 'xxx')
self.assertEqual(resp['bytes'], 3867)
@ -413,7 +286,25 @@ class TestFuncs(unittest.TestCase):
app = FakeApp()
req = Request.blank("/v1/a", environ={'swift.cache': FakeCache()})
get_account_info(req.environ, app, swift_source='MC')
self.assertEqual(app.sources, ['MC'])
self.assertEqual([e['swift.source'] for e in app.captured_envs],
['MC'])
def test_get_account_info_swift_owner(self):
app = FakeApp()
req = Request.blank("/v1/a", environ={'swift.cache': FakeCache()})
get_account_info(req.environ, app)
self.assertEqual([e['swift_owner'] for e in app.captured_envs],
[True])
def test_get_account_info_infocache(self):
app = FakeApp()
ic = {}
req = Request.blank("/v1/a", environ={'swift.cache': FakeCache(),
'swift.infocache': ic})
get_account_info(req.environ, app)
got_infocaches = [e['swift.infocache'] for e in app.captured_envs]
self.assertEqual(1, len(got_infocaches))
self.assertIs(ic, got_infocaches[0])
def test_get_account_info_no_cache(self):
app = FakeApp()
@ -424,7 +315,7 @@ class TestFuncs(unittest.TestCase):
self.assertEqual(resp['total_object_count'], 1000)
def test_get_account_info_cache(self):
# The original test that we prefer to preserve
# Works with fake apps that return ints in the headers
cached = {'status': 404,
'bytes': 3333,
'total_object_count': 10}
@ -435,7 +326,8 @@ class TestFuncs(unittest.TestCase):
self.assertEqual(resp['total_object_count'], 10)
self.assertEqual(resp['status'], 404)
# Here is a more realistic test
# Works with strings too, like you get when parsing HTTP headers
# that came in through a socket from the account server
cached = {'status': 404,
'bytes': '3333',
'container_count': '234',
@ -445,17 +337,17 @@ class TestFuncs(unittest.TestCase):
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, FakeApp())
self.assertEqual(resp['status'], 404)
self.assertEqual(resp['bytes'], '3333')
self.assertEqual(resp['bytes'], 3333)
self.assertEqual(resp['container_count'], 234)
self.assertEqual(resp['meta'], {})
self.assertEqual(resp['total_object_count'], '10')
self.assertEqual(resp['total_object_count'], 10)
def test_get_account_info_env(self):
cache_key = get_account_memcache_key("account")
env_key = 'swift.%s' % cache_key
req = Request.blank("/v1/account",
environ={env_key: {'bytes': 3867},
'swift.cache': FakeCache({})})
cache_key = get_cache_key("account")
req = Request.blank(
"/v1/account",
environ={'swift.infocache': {cache_key: {'bytes': 3867}},
'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, 'xxx')
self.assertEqual(resp['bytes'], 3867)
@ -464,10 +356,11 @@ class TestFuncs(unittest.TestCase):
'length': 3333,
'type': 'application/json',
'meta': {}}
env_key = get_object_env_key("account", "cont", "obj")
req = Request.blank("/v1/account/cont/obj",
environ={env_key: cached,
'swift.cache': FakeCache({})})
cache_key = get_cache_key("account", "cont", "obj")
req = Request.blank(
"/v1/account/cont/obj",
environ={'swift.infocache': {cache_key: cached},
'swift.cache': FakeCache({})})
resp = get_object_info(req.environ, 'xxx')
self.assertEqual(resp['length'], 3333)
self.assertEqual(resp['type'], 'application/json')
@ -722,6 +615,8 @@ class TestFuncs(unittest.TestCase):
self.assertEqual(handler.backend_headers['Range'], 'bytes=43-50')
self.assertRaises(HTTPException,
handler.fast_forward, 80)
self.assertRaises(exceptions.RangeAlreadyComplete,
handler.fast_forward, 8)
handler = GetOrHeadHandler(None, req, None, None, None, None,
{'Range': 'bytes=23-'})
@ -732,6 +627,41 @@ class TestFuncs(unittest.TestCase):
{'Range': 'bytes=-100'})
handler.fast_forward(20)
self.assertEqual(handler.backend_headers['Range'], 'bytes=-80')
self.assertRaises(HTTPException,
handler.fast_forward, 100)
self.assertRaises(exceptions.RangeAlreadyComplete,
handler.fast_forward, 80)
handler = GetOrHeadHandler(None, req, None, None, None, None,
{'Range': 'bytes=0-0'})
self.assertRaises(exceptions.RangeAlreadyComplete,
handler.fast_forward, 1)
def test_range_fast_forward_after_data_timeout(self):
req = Request.blank('/')
# We get a 200 and learn that it's a 1000-byte object, but receive 0
# bytes of data, so then we get a new node, fast_forward(0), and
# send out a new request. That new request must be for all 1000
# bytes.
handler = GetOrHeadHandler(None, req, None, None, None, None, {})
handler.learn_size_from_content_range(0, 999, 1000)
handler.fast_forward(0)
self.assertEqual(handler.backend_headers['Range'], 'bytes=0-999')
# Same story as above, but a 1-byte object so we can have our byte
# indices be 0.
handler = GetOrHeadHandler(None, req, None, None, None, None, {})
handler.learn_size_from_content_range(0, 0, 1)
handler.fast_forward(0)
self.assertEqual(handler.backend_headers['Range'], 'bytes=0-0')
# last 100 bytes
handler = GetOrHeadHandler(None, req, None, None, None, None,
{'Range': 'bytes=-100'})
handler.learn_size_from_content_range(900, 999, 1000)
handler.fast_forward(0)
self.assertEqual(handler.backend_headers['Range'], 'bytes=900-999')
def test_transfer_headers_with_sysmeta(self):
base = Controller(self.app)

14
test/unit/proxy/controllers/test_container.py
View File

@ -58,7 +58,7 @@ class TestContainerController(TestRingBase):
proxy_server.ContainerController):
def account_info(controller, *args, **kwargs):
patch_path = 'swift.proxy.controllers.base.get_info'
patch_path = 'swift.proxy.controllers.base.get_account_info'
with mock.patch(patch_path) as mock_get_info:
mock_get_info.return_value = dict(self.account_info)
return super(FakeAccountInfoContainerController,
@ -95,16 +95,20 @@ class TestContainerController(TestRingBase):
'Expected %s but got %s. Failed case: %s' %
(expected, resp.status_int, str(responses)))
def test_container_info_in_response_env(self):
def test_container_info_got_cached(self):
controller = proxy_server.ContainerController(self.app, 'a', 'c')
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, body='')):
req = Request.blank('/v1/a/c', {'PATH_INFO': '/v1/a/c'})
resp = controller.HEAD(req)
self.assertEqual(2, resp.status_int // 100)
self.assertTrue("swift.container/a/c" in resp.environ)
self.assertEqual(headers_to_container_info(resp.headers),
resp.environ['swift.container/a/c'])
# Make sure it's in both swift.infocache and memcache
self.assertIn("container/a/c", resp.environ['swift.infocache'])
self.assertEqual(
headers_to_container_info(resp.headers),
resp.environ['swift.infocache']['container/a/c'])
from_memcache = self.app.memcache.get('container/a/c')
self.assertTrue(from_memcache)
def test_swift_owner(self):
owner_headers = {

75
test/unit/proxy/controllers/test_obj.py
View File

@ -34,7 +34,8 @@ from swift.common import utils, swob, exceptions
from swift.common.header_key_dict import HeaderKeyDict
from swift.proxy import server as proxy_server
from swift.proxy.controllers import obj
from swift.proxy.controllers.base import get_info as _real_get_info
from swift.proxy.controllers.base import \
get_container_info as _real_get_container_info
from swift.common.storage_policy import POLICIES, ECDriverError, StoragePolicy
from test.unit import FakeRing, FakeMemcache, fake_http_connect, \
@ -76,7 +77,7 @@ def set_http_connect(*args, **kwargs):
class PatchedObjControllerApp(proxy_server.Application):
"""
This patch is just a hook over the proxy server's __call__ to ensure
that calls to get_info will return the stubbed value for
that calls to get_container_info will return the stubbed value for
container_info if it's a container info call.
"""
@ -85,16 +86,38 @@ class PatchedObjControllerApp(proxy_server.Application):
def __call__(self, *args, **kwargs):
def _fake_get_info(app, env, account, container=None, **kwargs):
if container:
if container in self.per_container_info:
return self.per_container_info[container]
return self.container_info
else:
return _real_get_info(app, env, account, container, **kwargs)
def _fake_get_container_info(env, app, swift_source=None):
_vrs, account, container, _junk = utils.split_path(
env['PATH_INFO'], 3, 4)
mock_path = 'swift.proxy.controllers.base.get_info'
with mock.patch(mock_path, new=_fake_get_info):
# Seed the cache with our container info so that the real
# get_container_info finds it.
ic = env.setdefault('swift.infocache', {})
cache_key = "container/%s/%s" % (account, container)
old_value = ic.get(cache_key)
# Copy the container info so we don't hand out a reference to a
# mutable thing that's set up only once at compile time. Nothing
# *should* mutate it, but it's better to be paranoid than wrong.
if container in self.per_container_info:
ic[cache_key] = self.per_container_info[container].copy()
else:
ic[cache_key] = self.container_info.copy()
real_info = _real_get_container_info(env, app, swift_source)
if old_value is None:
del ic[cache_key]
else:
ic[cache_key] = old_value
return real_info
with mock.patch('swift.proxy.server.get_container_info',
new=_fake_get_container_info), \
mock.patch('swift.proxy.controllers.base.get_container_info',
new=_fake_get_container_info):
return super(
PatchedObjControllerApp, self).__call__(*args, **kwargs)
@ -122,6 +145,7 @@ def make_footers_callback(body):
class BaseObjectControllerMixin(object):
container_info = {
'status': 200,
'write_acl': None,
'read_acl': None,
'storage_policy': None,
@ -142,8 +166,11 @@ class BaseObjectControllerMixin(object):
self.app = PatchedObjControllerApp(
None, FakeMemcache(), account_ring=FakeRing(),
container_ring=FakeRing(), logger=self.logger)
# you can over-ride the container_info just by setting it on the app
# (see PatchedObjControllerApp for details)
self.app.container_info = dict(self.container_info)
# default policy and ring references
self.policy = POLICIES.default
self.obj_ring = self.policy.object_ring
@ -1136,31 +1163,6 @@ class TestReplicatedObjControllerVariousReplicas(BaseObjectControllerMixin,
controller_cls = obj.ReplicatedObjectController
@patch_policies(legacy_only=True)
class TestObjControllerLegacyCache(TestReplicatedObjController):
"""
This test pretends like memcache returned a stored value that should
resemble whatever "old" format. It catches KeyErrors you'd get if your
code was expecting some new format during a rolling upgrade.
"""
# in this case policy_index is missing
container_info = {
'read_acl': None,
'write_acl': None,
'sync_key': None,
'versions': None,
}
def test_invalid_storage_policy_cache(self):
self.app.container_info['storage_policy'] = 1
for method in ('GET', 'HEAD', 'POST', 'PUT', 'COPY'):
req = swob.Request.blank('/v1/a/c/o', method=method)
with set_http_connect():
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
class StubResponse(object):
def __init__(self, status, body='', headers=None):
@ -1234,6 +1236,7 @@ def capture_http_requests(get_response):
@patch_policies(with_ec_default=True)
class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
container_info = {
'status': 200,
'read_acl': None,
'write_acl': None,
'sync_key': None,

137
test/unit/proxy/test_server.py
View File

@ -70,8 +70,8 @@ from swift.common import utils, constraints
from swift.common.utils import mkdirs, NullLogger
from swift.common.wsgi import monkey_patch_mimetools, loadapp
from swift.proxy.controllers import base as proxy_base
from swift.proxy.controllers.base import get_container_memcache_key, \
get_account_memcache_key, cors_validation, _get_info_cache
from swift.proxy.controllers.base import get_cache_key, cors_validation, \
get_account_info, get_container_info
import swift.proxy.controllers
import swift.proxy.controllers.obj
from swift.common.header_key_dict import HeaderKeyDict
@ -285,14 +285,14 @@ class TestController(unittest.TestCase):
self.controller.account_info(self.account)
self.assertEqual(count, 123)
with save_globals():
cache_key = get_account_memcache_key(self.account)
cache_key = get_cache_key(self.account)
account_info = {'status': 200, 'container_count': 1234}
self.memcache.set(cache_key, account_info)
partition, nodes, count = \
self.controller.account_info(self.account)
self.assertEqual(count, 1234)
with save_globals():
cache_key = get_account_memcache_key(self.account)
cache_key = get_cache_key(self.account)
account_info = {'status': 200, 'container_count': '1234'}
self.memcache.set(cache_key, account_info)
partition, nodes, count = \
@ -320,8 +320,9 @@ class TestController(unittest.TestCase):
# Test the internal representation in memcache
# 'container_count' changed from int to str
cache_key = get_account_memcache_key(self.account)
cache_key = get_cache_key(self.account)
container_info = {'status': 200,
'account_really_exists': True,
'container_count': '12345',
'total_object_count': None,
'bytes': None,
@ -347,7 +348,7 @@ class TestController(unittest.TestCase):
# Test the internal representation in memcache
# 'container_count' changed from 0 to None
cache_key = get_account_memcache_key(self.account)
cache_key = get_cache_key(self.account)
account_info = {'status': 404,
'container_count': None, # internally keep None
'total_object_count': None,
@ -424,8 +425,7 @@ class TestController(unittest.TestCase):
self.account, self.container, self.request)
self.check_container_info_return(ret)
cache_key = get_container_memcache_key(self.account,
self.container)
cache_key = get_cache_key(self.account, self.container)
cache_value = self.memcache.get(cache_key)
self.assertTrue(isinstance(cache_value, dict))
self.assertEqual(200, cache_value.get('status'))
@ -447,8 +447,7 @@ class TestController(unittest.TestCase):
self.account, self.container, self.request)
self.check_container_info_return(ret, True)
cache_key = get_container_memcache_key(self.account,
self.container)
cache_key = get_cache_key(self.account, self.container)
cache_value = self.memcache.get(cache_key)
self.assertTrue(isinstance(cache_value, dict))
self.assertEqual(404, cache_value.get('status'))
@ -463,8 +462,7 @@ class TestController(unittest.TestCase):
self.account, self.container, self.request)
self.check_container_info_return(ret, True)
cache_key = get_container_memcache_key(self.account,
self.container)
cache_key = get_cache_key(self.account, self.container)
cache_value = self.memcache.get(cache_key)
self.assertTrue(isinstance(cache_value, dict))
self.assertEqual(404, cache_value.get('status'))
@ -490,7 +488,32 @@ class TestController(unittest.TestCase):
test(404, 507, 503)
test(503, 503, 503)
def test_get_info_cache_returns_values_as_strings(self):
def test_get_account_info_returns_values_as_strings(self):
app = mock.MagicMock()
app.memcache = mock.MagicMock()
app.memcache.get = mock.MagicMock()
app.memcache.get.return_value = {
u'foo': u'\u2603',
u'meta': {u'bar': u'\u2603'},
u'sysmeta': {u'baz': u'\u2603'}}
env = {'PATH_INFO': '/v1/a'}
ai = get_account_info(env, app)
# Test info is returned as strings
self.assertEqual(ai.get('foo'), '\xe2\x98\x83')
self.assertTrue(isinstance(ai.get('foo'), str))
# Test info['meta'] is returned as strings
m = ai.get('meta', {})
self.assertEqual(m.get('bar'), '\xe2\x98\x83')
self.assertTrue(isinstance(m.get('bar'), str))
# Test info['sysmeta'] is returned as strings
m = ai.get('sysmeta', {})
self.assertEqual(m.get('baz'), '\xe2\x98\x83')
self.assertTrue(isinstance(m.get('baz'), str))
def test_get_container_info_returns_values_as_strings(self):
app = mock.MagicMock()
app.memcache = mock.MagicMock()
app.memcache.get = mock.MagicMock()
@ -499,25 +522,25 @@ class TestController(unittest.TestCase):
u'meta': {u'bar': u'\u2603'},
u'sysmeta': {u'baz': u'\u2603'},
u'cors': {u'expose_headers': u'\u2603'}}
env = {}
r = _get_info_cache(app, env, 'account', 'container')
env = {'PATH_INFO': '/v1/a/c'}
ci = get_container_info(env, app)
# Test info is returned as strings
self.assertEqual(r.get('foo'), '\xe2\x98\x83')
self.assertTrue(isinstance(r.get('foo'), str))
self.assertEqual(ci.get('foo'), '\xe2\x98\x83')
self.assertTrue(isinstance(ci.get('foo'), str))
# Test info['meta'] is returned as strings
m = r.get('meta', {})
m = ci.get('meta', {})
self.assertEqual(m.get('bar'), '\xe2\x98\x83')
self.assertTrue(isinstance(m.get('bar'), str))
# Test info['sysmeta'] is returned as strings
m = r.get('sysmeta', {})
m = ci.get('sysmeta', {})
self.assertEqual(m.get('baz'), '\xe2\x98\x83')
self.assertTrue(isinstance(m.get('baz'), str))
# Test info['cors'] is returned as strings
m = r.get('cors', {})
m = ci.get('cors', {})
self.assertEqual(m.get('expose_headers'), '\xe2\x98\x83')
self.assertTrue(isinstance(m.get('expose_headers'), str))
@ -6141,22 +6164,23 @@ class TestContainerController(unittest.TestCase):
res = controller.HEAD(req)
self.assertEqual(res.status[:len(str(expected))],
str(expected))
infocache = res.environ.get('swift.infocache', {})
if expected < 400:
self.assertTrue('x-works' in res.headers)
self.assertIn('x-works', res.headers)
self.assertEqual(res.headers['x-works'], 'yes')
if c_expected:
self.assertTrue('swift.container/a/c' in res.environ)
self.assertIn('container/a/c', infocache)
self.assertEqual(
res.environ['swift.container/a/c']['status'],
infocache['container/a/c']['status'],
c_expected)
else:
self.assertTrue('swift.container/a/c' not in res.environ)
self.assertNotIn('container/a/c', infocache)
if a_expected:
self.assertTrue('swift.account/a' in res.environ)
self.assertEqual(res.environ['swift.account/a']['status'],
self.assertIn('account/a', infocache)
self.assertEqual(infocache['account/a']['status'],
a_expected)
else:
self.assertTrue('swift.account/a' not in res.environ)
self.assertNotIn('account/a', res.environ)
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
@ -6165,25 +6189,26 @@ class TestContainerController(unittest.TestCase):
res = controller.GET(req)
self.assertEqual(res.status[:len(str(expected))],
str(expected))
infocache = res.environ.get('swift.infocache', {})
if expected < 400:
self.assertTrue('x-works' in res.headers)
self.assertEqual(res.headers['x-works'], 'yes')
if c_expected:
self.assertTrue('swift.container/a/c' in res.environ)
self.assertIn('container/a/c', infocache)
self.assertEqual(
res.environ['swift.container/a/c']['status'],
infocache['container/a/c']['status'],
c_expected)
else:
self.assertTrue('swift.container/a/c' not in res.environ)
self.assertNotIn('container/a/c', infocache)
if a_expected:
self.assertTrue('swift.account/a' in res.environ)
self.assertEqual(res.environ['swift.account/a']['status'],
self.assertIn('account/a', infocache)
self.assertEqual(infocache['account/a']['status'],
a_expected)
else:
self.assertTrue('swift.account/a' not in res.environ)
self.assertNotIn('account/a', infocache)
# In all the following tests cache 200 for account
# return and ache vary for container
# return 200 and cache 200 for and container
# return and cache vary for container
# return 200 and cache 200 for account and container
test_status_map((200, 200, 404, 404), 200, 200, 200)
test_status_map((200, 200, 500, 404), 200, 200, 200)
# return 304 don't cache container
@ -6195,12 +6220,13 @@ class TestContainerController(unittest.TestCase):
test_status_map((200, 500, 500, 500), 503, None, 200)
self.assertFalse(self.app.account_autocreate)
# In all the following tests cache 404 for account
# return 404 (as account is not found) and don't cache container
test_status_map((404, 404, 404), 404, None, 404)
# This should make no difference
# cache a 204 for the account because it's sort of like it
# exists
self.app.account_autocreate = True
test_status_map((404, 404, 404), 404, None, 404)
test_status_map((404, 404, 404), 404, None, 204)
def test_PUT_policy_headers(self):
backend_requests = []
@ -6786,14 +6812,13 @@ class TestContainerController(unittest.TestCase):
def test_GET_no_content(self):
with save_globals():
set_http_connect(200, 204, 204, 204)
controller = proxy_server.ContainerController(self.app, 'account',
'container')
controller = proxy_server.ContainerController(self.app, 'a', 'c')
req = Request.blank('/v1/a/c')
self.app.update_request(req)
res = controller.GET(req)
self.assertEqual(res.status_int, 204)
self.assertEqual(
res.environ['swift.container/a/c']['status'], 204)
ic = res.environ['swift.infocache']
self.assertEqual(ic['container/a/c']['status'], 204)
self.assertEqual(res.content_length, 0)
self.assertTrue('transfer-encoding' not in res.headers)
@ -6805,13 +6830,14 @@ class TestContainerController(unittest.TestCase):
return HTTPUnauthorized(request=req)
with save_globals():
set_http_connect(200, 201, 201, 201)
controller = proxy_server.ContainerController(self.app, 'account',
'container')
controller = proxy_server.ContainerController(self.app, 'a', 'c')
req = Request.blank('/v1/a/c')
req.environ['swift.authorize'] = authorize
self.app.update_request(req)
res = controller.GET(req)
self.assertEqual(res.environ['swift.container/a/c']['status'], 201)
self.assertEqual(
res.environ['swift.infocache']['container/a/c']['status'],
201)
self.assertTrue(called[0])
def test_HEAD_calls_authorize(self):
@ -6822,8 +6848,7 @@ class TestContainerController(unittest.TestCase):
return HTTPUnauthorized(request=req)
with save_globals():
set_http_connect(200, 201, 201, 201)
controller = proxy_server.ContainerController(self.app, 'account',
'container')
controller = proxy_server.ContainerController(self.app, 'a', 'c')
req = Request.blank('/v1/a/c', {'REQUEST_METHOD': 'HEAD'})
req.environ['swift.authorize'] = authorize
self.app.update_request(req)
@ -7279,16 +7304,18 @@ class TestAccountController(unittest.TestCase):
self.app.update_request(req)
res = method(req)
self.assertEqual(res.status_int, expected)
infocache = res.environ.get('swift.infocache', {})
if env_expected:
self.assertEqual(res.environ['swift.account/a']['status'],
self.assertEqual(infocache['account/a']['status'],
env_expected)
set_http_connect(*statuses)
req = Request.blank('/v1/a/', {})
self.app.update_request(req)
res = method(req)
infocache = res.environ.get('swift.infocache', {})
self.assertEqual(res.status_int, expected)
if env_expected:
self.assertEqual(res.environ['swift.account/a']['status'],
self.assertEqual(infocache['account/a']['status'],
env_expected)
def test_OPTIONS(self):
@ -7333,7 +7360,7 @@ class TestAccountController(unittest.TestCase):
def test_GET(self):
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
controller = proxy_server.AccountController(self.app, 'a')
# GET returns after the first successful call to an Account Server
self.assert_status_map(controller.GET, (200,), 200, 200)
self.assert_status_map(controller.GET, (503, 200), 200, 200)
@ -7355,7 +7382,7 @@ class TestAccountController(unittest.TestCase):
def test_GET_autocreate(self):
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
controller = proxy_server.AccountController(self.app, 'a')
self.app.memcache = FakeMemcacheReturnsNone()
self.assertFalse(self.app.account_autocreate)
# Repeat the test for autocreate = False and 404 by all
@ -7380,7 +7407,7 @@ class TestAccountController(unittest.TestCase):
def test_HEAD(self):
# Same behaviour as GET
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
controller = proxy_server.AccountController(self.app, 'a')
self.assert_status_map(controller.HEAD, (200,), 200, 200)
self.assert_status_map(controller.HEAD, (503, 200), 200, 200)
self.assert_status_map(controller.HEAD, (503, 503, 200), 200, 200)
@ -7398,7 +7425,7 @@ class TestAccountController(unittest.TestCase):
def test_HEAD_autocreate(self):
# Same behaviour as GET
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
controller = proxy_server.AccountController(self.app, 'a')
self.app.memcache = FakeMemcacheReturnsNone()
self.assertFalse(self.app.account_autocreate)
self.assert_status_map(controller.HEAD,
@ -7414,7 +7441,7 @@ class TestAccountController(unittest.TestCase):
def test_POST_autocreate(self):
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
controller = proxy_server.AccountController(self.app, 'a')
self.app.memcache = FakeMemcacheReturnsNone()
# first test with autocreate being False
self.assertFalse(self.app.account_autocreate)
@ -7436,7 +7463,7 @@ class TestAccountController(unittest.TestCase):
def test_POST_autocreate_with_sysmeta(self):
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
controller = proxy_server.AccountController(self.app, 'a')
self.app.memcache = FakeMemcacheReturnsNone()
# first test with autocreate being False
self.assertFalse(self.app.account_autocreate)

147
test/unit/proxy/test_sysmeta.py
View File

@ -19,6 +19,8 @@ import unittest
import os
from tempfile import mkdtemp
import shutil
from swift.common.middleware.copy import ServerSideCopyMiddleware
from swift.common.storage_policy import StoragePolicy
from swift.common.swob import Request
from swift.common.utils import mkdirs, split_path
@ -134,6 +136,7 @@ class TestObjectSysmeta(unittest.TestCase):
logger=debug_logger('proxy-ut'),
account_ring=FakeRing(replicas=1),
container_ring=FakeRing(replicas=1))
self.copy_app = ServerSideCopyMiddleware(self.app, {})
monkey_patch_mimetools()
self.tmpdir = mkdtemp()
self.testdir = os.path.join(self.tmpdir,
@ -253,7 +256,7 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertInHeaders(resp, self.new_meta_headers)
self._assertNotInHeaders(resp, self.original_meta_headers_2)
def test_sysmeta_not_updated_by_POST(self):
def _test_sysmeta_not_updated_by_POST(self, app):
# check sysmeta is not changed by a POST but user meta is replaced
path = '/v1/a/c/o'
@ -261,7 +264,7 @@ class TestObjectSysmeta(unittest.TestCase):
hdrs = dict(self.original_sysmeta_headers_1)
hdrs.update(self.original_meta_headers_1)
req = Request.blank(path, environ=env, headers=hdrs, body='x')
resp = req.get_response(self.app)
resp = req.get_response(app)
self._assertStatus(resp, 201)
env = {'REQUEST_METHOD': 'POST'}
@ -271,11 +274,11 @@ class TestObjectSysmeta(unittest.TestCase):
hdrs.update(self.new_meta_headers)
hdrs.update(self.bad_headers)
req = Request.blank(path, environ=env, headers=hdrs)
resp = req.get_response(self.app)
resp = req.get_response(app)
self._assertStatus(resp, 202)
req = Request.blank(path, environ={})
resp = req.get_response(self.app)
resp = req.get_response(app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.original_sysmeta_headers_1)
self._assertNotInHeaders(resp, self.new_sysmeta_headers)
@ -288,17 +291,121 @@ class TestObjectSysmeta(unittest.TestCase):
hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.bad_headers)
req = Request.blank(path, environ=env, headers=hdrs, body='x')
resp = req.get_response(self.app)
resp = req.get_response(app)
self._assertStatus(resp, 201)
req = Request.blank(path, environ={})
resp = req.get_response(self.app)
resp = req.get_response(app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_sysmeta_headers)
self._assertInHeaders(resp, self.new_sysmeta_headers)
self._assertNotInHeaders(resp, self.original_sysmeta_headers_2)
def test_transient_sysmeta_replaced_by_PUT_or_POST(self):
def test_sysmeta_not_updated_by_POST(self):
# test fast-post by issuing requests to the proxy app
self._test_sysmeta_not_updated_by_POST(self.app)
def test_sysmeta_not_updated_by_POST_as_copy(self):
# test post-as-copy by issuing requests to the copy middleware app
self.copy_app.object_post_as_copy = True
self._test_sysmeta_not_updated_by_POST(self.copy_app)
def test_sysmeta_updated_by_COPY(self):
# check sysmeta is updated by a COPY in same way as user meta by
# issuing requests to the copy middleware app
path = '/v1/a/c/o'
dest = '/c/o2'
env = {'REQUEST_METHOD': 'PUT'}
hdrs = dict(self.original_sysmeta_headers_1)
hdrs.update(self.original_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_1)
hdrs.update(self.original_meta_headers_2)
hdrs.update(self.original_transient_sysmeta_headers_1)
hdrs.update(self.original_transient_sysmeta_headers_2)
req = Request.blank(path, environ=env, headers=hdrs, body='x')
resp = req.get_response(self.copy_app)
self._assertStatus(resp, 201)
env = {'REQUEST_METHOD': 'COPY'}
hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.changed_meta_headers)
hdrs.update(self.new_meta_headers)
hdrs.update(self.changed_transient_sysmeta_headers)
hdrs.update(self.new_transient_sysmeta_headers)
hdrs.update(self.bad_headers)
hdrs.update({'Destination': dest})
req = Request.blank(path, environ=env, headers=hdrs)
resp = req.get_response(self.copy_app)
self._assertStatus(resp, 201)
self._assertInHeaders(resp, self.changed_sysmeta_headers)
self._assertInHeaders(resp, self.new_sysmeta_headers)
self._assertInHeaders(resp, self.original_sysmeta_headers_2)
self._assertInHeaders(resp, self.changed_meta_headers)
self._assertInHeaders(resp, self.new_meta_headers)
self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertInHeaders(resp, self.changed_transient_sysmeta_headers)
self._assertInHeaders(resp, self.new_transient_sysmeta_headers)
self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers)
req = Request.blank('/v1/a/c/o2', environ={})
resp = req.get_response(self.copy_app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_sysmeta_headers)
self._assertInHeaders(resp, self.new_sysmeta_headers)
self._assertInHeaders(resp, self.original_sysmeta_headers_2)
self._assertInHeaders(resp, self.changed_meta_headers)
self._assertInHeaders(resp, self.new_meta_headers)
self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertInHeaders(resp, self.changed_transient_sysmeta_headers)
self._assertInHeaders(resp, self.new_transient_sysmeta_headers)
self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers)
def test_sysmeta_updated_by_COPY_from(self):
# check sysmeta is updated by a PUT with x-copy-from in same way as
# user meta by issuing requests to the copy middleware app
path = '/v1/a/c/o'
env = {'REQUEST_METHOD': 'PUT'}
hdrs = dict(self.original_sysmeta_headers_1)
hdrs.update(self.original_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_1)
hdrs.update(self.original_meta_headers_2)
req = Request.blank(path, environ=env, headers=hdrs, body='x')
resp = req.get_response(self.copy_app)
self._assertStatus(resp, 201)
env = {'REQUEST_METHOD': 'PUT'}
hdrs = dict(self.changed_sysmeta_headers)
hdrs.update(self.new_sysmeta_headers)
hdrs.update(self.changed_meta_headers)
hdrs.update(self.new_meta_headers)
hdrs.update(self.bad_headers)
hdrs.update({'X-Copy-From': '/c/o'})
req = Request.blank('/v1/a/c/o2', environ=env, headers=hdrs, body='')
resp = req.get_response(self.copy_app)
self._assertStatus(resp, 201)
self._assertInHeaders(resp, self.changed_sysmeta_headers)
self._assertInHeaders(resp, self.new_sysmeta_headers)
self._assertInHeaders(resp, self.original_sysmeta_headers_2)
self._assertInHeaders(resp, self.changed_meta_headers)
self._assertInHeaders(resp, self.new_meta_headers)
self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers)
req = Request.blank('/v1/a/c/o2', environ={})
resp = req.get_response(self.copy_app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.changed_sysmeta_headers)
self._assertInHeaders(resp, self.new_sysmeta_headers)
self._assertInHeaders(resp, self.original_sysmeta_headers_2)
self._assertInHeaders(resp, self.changed_meta_headers)
self._assertInHeaders(resp, self.new_meta_headers)
self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertNotInHeaders(resp, self.bad_headers)
def _test_transient_sysmeta_replaced_by_PUT_or_POST(self, app):
# check transient_sysmeta is replaced en-masse by a POST
path = '/v1/a/c/o'
@ -307,17 +414,17 @@ class TestObjectSysmeta(unittest.TestCase):
hdrs.update(self.original_transient_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_1)
req = Request.blank(path, environ=env, headers=hdrs, body='x')
resp = req.get_response(self.app)
resp = req.get_response(app)
self._assertStatus(resp, 201)
req = Request.blank(path, environ={})
resp = req.get_response(self.app)
resp = req.get_response(app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.original_transient_sysmeta_headers_1)
self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2)
self._assertInHeaders(resp, self.original_meta_headers_1)
info = get_object_info(req.environ, self.app)
info = get_object_info(req.environ, app)
self.assertEqual(2, len(info.get('transient_sysmeta', ())))
self.assertEqual({'testa': 'A', 'testb': 'B'},
info['transient_sysmeta'])
@ -327,11 +434,11 @@ class TestObjectSysmeta(unittest.TestCase):
hdrs = dict(self.changed_transient_sysmeta_headers)
hdrs.update(self.new_transient_sysmeta_headers)
req = Request.blank(path, environ=env, headers=hdrs)
resp = req.get_response(self.app)
resp = req.get_response(app)
self._assertStatus(resp, 202)
req = Request.blank(path, environ={})
resp = req.get_response(self.app)
resp = req.get_response(app)
self._assertStatus(resp, 200)
self._assertNotInHeaders(resp, self.original_meta_headers_2)
self._assertInHeaders(resp, self.changed_transient_sysmeta_headers)
@ -339,7 +446,7 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertNotInHeaders(resp,
self.original_transient_sysmeta_headers_2)
info = get_object_info(req.environ, self.app)
info = get_object_info(req.environ, app)
self.assertEqual(2, len(info.get('transient_sysmeta', ())))
self.assertEqual({'testa': 'changed_A', 'testc': 'C'},
info['transient_sysmeta'])
@ -349,11 +456,11 @@ class TestObjectSysmeta(unittest.TestCase):
hdrs = dict(self.original_transient_sysmeta_headers_2)
hdrs.update(self.original_meta_headers_2)
req = Request.blank(path, environ=env, headers=hdrs, body='x')
resp = req.get_response(self.app)
resp = req.get_response(app)
self._assertStatus(resp, 201)
req = Request.blank(path, environ={})
resp = req.get_response(self.app)
resp = req.get_response(app)
self._assertStatus(resp, 200)
self._assertInHeaders(resp, self.original_meta_headers_2)
self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2)
@ -362,6 +469,14 @@ class TestObjectSysmeta(unittest.TestCase):
self._assertNotInHeaders(resp, self.changed_transient_sysmeta_headers)
self._assertNotInHeaders(resp, self.new_transient_sysmeta_headers)
info = get_object_info(req.environ, self.app)
info = get_object_info(req.environ, app)
self.assertEqual(1, len(info.get('transient_sysmeta', ())))
self.assertEqual({'testb': 'B'}, info['transient_sysmeta'])
def test_transient_sysmeta_replaced_by_PUT_or_POST(self):
self._test_transient_sysmeta_replaced_by_PUT_or_POST(self.app)
def test_transient_sysmeta_replaced_by_PUT_or_POST_as_copy(self):
# test post-as-copy by issuing requests to the copy middleware app
self.copy_app.object_post_as_copy = True
self._test_transient_sysmeta_replaced_by_PUT_or_POST(self.copy_app)

11
tox.ini
View File

@ -76,7 +76,14 @@ commands = bandit -c bandit.yaml -r swift bin -n 5 -p gate
# H404: multi line docstring should start without a leading new line
# H405: multi line docstring summary not separated with an empty line
# H501: Do not use self.__dict__ for string formatting
# H703: Multiple positional placeholders
ignore = F812,H101,H202,H233,H301,H306,H401,H403,H404,H405,H501,H703
ignore = F812,H101,H202,H233,H301,H306,H401,H403,H404,H405,H501
exclude = .venv,.tox,dist,*egg
show-source = True
[testenv:bindep]
# Do not install any requirements. We want this to be fast and work even if
# system dependencies are missing, since it's used to tell you what system
# dependencies are missing! This also means that bindep must be installed
# separately, outside of the requirements files.
deps = bindep
commands = bindep test