diff --git a/MANIFEST.in b/MANIFEST.in index 87eb0c9c96..4a65073dba 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,7 +6,7 @@ include tox.ini include requirements.txt test-requirements.txt graft doc graft etc -graft locale +graft swift/locale graft test/functional graft test/probe graft test/unit diff --git a/README.rst b/README.rst index 984d160e21..2c2831dede 100644 --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ Getting Started Swift is part of OpenStack and follows the code contribution, review, and testing processes common to all OpenStack projects. If you would like to start contributing, check out these -`notes `__ to help you get started. +`notes `__ to help you get started. The best place to get started is the `"SAIO - Swift All In One" `__. diff --git a/doc/source/associated_projects.rst b/doc/source/associated_projects.rst index b92dc4ed21..46e0238564 100644 --- a/doc/source/associated_projects.rst +++ b/doc/source/associated_projects.rst @@ -107,7 +107,6 @@ Other * `Glance `_ - Provides services for discovering, registering, and retrieving virtual machine images (for OpenStack Compute [Nova], for example). * `Better Staticweb `_ - Makes swift containers accessible by default. -* `Swiftsync `_ - A massive syncer between two swift clusters. * `Django Swiftbrowser `_ - Simple Django web app to access OpenStack Swift. * `Swift-account-stats `_ - Swift-account-stats is a tool to report statistics on Swift usage at tenant and global levels. * `PyECLib `_ - High Level Erasure Code library used by Swift diff --git a/doc/source/development_guidelines.rst b/doc/source/development_guidelines.rst index fd3607015f..6f0012c35f 100644 --- a/doc/source/development_guidelines.rst +++ b/doc/source/development_guidelines.rst @@ -27,6 +27,12 @@ To execute the tests: pip install tox +* Generate list of distribution packages to install for testing:: + + tox -e bindep + + Now install these packages using your distribution package manager + like apt-get, dnf, yum, or zypper. * Run Tox from the root of the swift repo:: diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index bca218cad5..1d497a526f 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -591,3 +591,17 @@ doesn't work, here are some good starting places to look for issues: you check that you can ``GET`` account, use ``sudo service memcached status`` and check if memcache is running. If memcache is not running, start it using ``sudo service memcached start``. Once memcache is running, rerun ``GET`` account. + +------------ +Known Issues +------------ + +Listed here are some "gotcha's" that you may run into when using or testing your SAIO: + +#. fallocate_reserve - in most cases a SAIO doesn't have a very large XFS partition + so having fallocate enabled and fallocate_reserve set can cause issues, specifically + when trying to run the functional tests. For this reason fallocate has been turned + off on the object-servers in the SAIO. If you want to play with the fallocate_reserve + settings then know that functional tests will fail unless you change the max_file_size + constraint to something more reasonable then the default (5G). Ideally you'd make + it 1/4 of your XFS file system size so the tests can pass. diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index ba8790821b..8308c130ae 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -15,6 +15,7 @@ Swift is written in Python and has these dependencies: * rsync 3.0 * The Python packages listed in `the requirements file `_ * Testing additionally requires `the test dependencies `_ +* Testing requires `these distribution packages `_ There is no current support for Python 3. diff --git a/other-requirements.txt b/other-requirements.txt new file mode 100644 index 0000000000..2fef68fdd8 --- /dev/null +++ b/other-requirements.txt @@ -0,0 +1,17 @@ +# This is a cross-platform list tracking distribution packages needed by tests; +# see http://docs.openstack.org/infra/bindep/ for additional information. + +build-essential [platform:dpkg] +gcc [platform:rpm] +gettext +liberasurecode-dev [platform:dpkg] +liberasurecode-devel [platform:rpm] +libffi-dev [platform:dpkg] +libffi-devel [platform:rpm] +memcached +python-dev [platform:dpkg] +python-devel [platform:rpm] +rsync +xfsprogs +libssl-dev [platform:dpkg] +openssl-devel [platform:rpm] diff --git a/swift/common/container_sync_realms.py b/swift/common/container_sync_realms.py index 7b441da9de..2c4c944add 100644 --- a/swift/common/container_sync_realms.py +++ b/swift/common/container_sync_realms.py @@ -57,7 +57,8 @@ class ContainerSyncRealms(object): log_func = self.logger.debug else: log_func = self.logger.error - log_func(_('Could not load %r: %s'), self.conf_path, err) + log_func(_('Could not load %(conf)r: %(error)s') % { + 'conf': self.conf_path, 'error': err}) else: if mtime != self.conf_path_mtime: self.conf_path_mtime = mtime @@ -66,7 +67,8 @@ class ContainerSyncRealms(object): conf.read(self.conf_path) except configparser.ParsingError as err: self.logger.error( - _('Could not load %r: %s'), self.conf_path, err) + _('Could not load %(conf)r: %(error)s') + % {'conf': self.conf_path, 'error': err}) else: try: self.mtime_check_interval = conf.getint( @@ -79,8 +81,9 @@ class ContainerSyncRealms(object): now + self.mtime_check_interval except (configparser.ParsingError, ValueError) as err: self.logger.error( - _('Error in %r with mtime_check_interval: %s'), - self.conf_path, err) + _('Error in %(conf)r with ' + 'mtime_check_interval: %(error)s') + % {'conf': self.conf_path, 'error': err}) realms = {} for section in conf.sections(): realm = {} diff --git a/swift/common/crypto_utils.py b/swift/common/crypto_utils.py index ce042e7f06..4740d5229f 100644 --- a/swift/common/crypto_utils.py +++ b/swift/common/crypto_utils.py @@ -67,7 +67,8 @@ class CryptoWSGIContext(WSGIContext): self.logger.exception(_("Did not get a keys dict")) except ValueError as e: # don't include the key in any messages! - self.logger.exception(_("Bad key for %r: %s") % (name, str(e))) + self.logger.exception(_("Bad key for %(name)r: %(err)s") % + {'name': name, 'err': e}) raise HTTPInternalServerError( "Unable to retrieve encryption keys.") diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index d4abf25efe..7115a8afe4 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -525,10 +525,13 @@ class Replicator(Daemon): if shouldbehere: shouldbehere = bool([n for n in nodes if n['id'] == node_id]) # See Footnote [1] for an explanation of the repl_nodes assignment. - i = 0 - while i < len(nodes) and nodes[i]['id'] != node_id: - i += 1 - repl_nodes = nodes[i + 1:] + nodes[:i] + if len(nodes) > 1: + i = 0 + while i < len(nodes) and nodes[i]['id'] != node_id: + i += 1 + repl_nodes = nodes[i + 1:] + nodes[:i] + else: # Special case if using only a single replica + repl_nodes = nodes more_nodes = self.ring.get_more_nodes(int(partition)) if not local_dev: # Check further if local device is a handoff node @@ -563,7 +566,7 @@ class Replicator(Daemon): except (Exception, Timeout): self.logger.exception('UNHANDLED EXCEPTION: in post replicate ' 'hook for %s', broker.db_file) - if not shouldbehere and all(responses): + if not shouldbehere and responses and all(responses): # If the db shouldn't be on this node and has been successfully # synced to all of its peers, it can be removed. if not self.delete_db(broker): diff --git a/swift/common/middleware/copy.py b/swift/common/middleware/copy.py index 46f4723f84..c5b7a1f591 100644 --- a/swift/common/middleware/copy.py +++ b/swift/common/middleware/copy.py @@ -102,7 +102,7 @@ accounts: ------------------- Large Object Copy ------------------- -The best option to copy a large option is to copy segments individually. +The best option to copy a large object is to copy segments individually. To copy the manifest object of a large object, add the query parameter to the copy request:: @@ -425,7 +425,9 @@ class ServerSideCopyMiddleware(object): # Existing sys and user meta of source object is added to response # headers in addition to the new ones. for k, v in sink_req.headers.items(): - if is_sys_or_user_meta('object', k) or k.lower() == 'x-delete-at': + if (is_sys_or_user_meta('object', k) or + k.lower() == 'x-delete-at' or + is_object_transient_sysmeta(k)): resp_headers[k] = v return resp_headers diff --git a/swift/common/middleware/keystoneauth.py b/swift/common/middleware/keystoneauth.py index 651aeacfbb..ccdd2a8ba9 100644 --- a/swift/common/middleware/keystoneauth.py +++ b/swift/common/middleware/keystoneauth.py @@ -287,7 +287,8 @@ class KeystoneAuth(object): def _get_project_domain_id(self, environ): info = get_account_info(environ, self.app, 'KS') domain_id = info.get('sysmeta', {}).get('project-domain-id') - exists = is_success(info.get('status', 0)) + exists = (is_success(info.get('status', 0)) + and info.get('account_really_exists', True)) return exists, domain_id def _set_project_domain_id(self, req, path_parts, env_identity): diff --git a/swift/common/swob.py b/swift/common/swob.py index 60970e419c..5bdfd63ef0 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -1086,6 +1086,7 @@ class Response(object): content_range = _header_property('content-range') etag = _resp_etag_property() status = _resp_status_property() + status_int = None body = _resp_body_property() host_url = _host_url_property() last_modified = _datetime_property('last-modified') diff --git a/swift/common/utils.py b/swift/common/utils.py index a33df51ed8..1d12b96495 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -596,7 +596,8 @@ class FileLikeIter(object): class FallocateWrapper(object): def __init__(self, noop=False): - if noop: + self.noop = noop + if self.noop: self.func_name = 'posix_fallocate' self.fallocate = noop_libc_function return @@ -614,16 +615,18 @@ class FallocateWrapper(object): def __call__(self, fd, mode, offset, length): """The length parameter must be a ctypes.c_uint64.""" - if FALLOCATE_RESERVE > 0: - st = os.fstatvfs(fd) - free = st.f_frsize * st.f_bavail - length.value - if FALLOCATE_IS_PERCENT: - free = (float(free) / float(st.f_frsize * st.f_blocks)) * 100 - if float(free) <= float(FALLOCATE_RESERVE): - raise OSError( - errno.ENOSPC, - 'FALLOCATE_RESERVE fail %s <= %s' % (free, - FALLOCATE_RESERVE)) + if not self.noop: + if FALLOCATE_RESERVE > 0: + st = os.fstatvfs(fd) + free = st.f_frsize * st.f_bavail - length.value + if FALLOCATE_IS_PERCENT: + free = \ + (float(free) / float(st.f_frsize * st.f_blocks)) * 100 + if float(free) <= float(FALLOCATE_RESERVE): + raise OSError( + errno.ENOSPC, + 'FALLOCATE_RESERVE fail %s <= %s' % + (free, FALLOCATE_RESERVE)) args = { 'fallocate': (fd, mode, offset, length), 'posix_fallocate': (fd, offset, length) @@ -2671,7 +2674,8 @@ def validate_sync_to(value, allowed_sync_hosts, realms_conf): endpoint = realms_conf.endpoint(realm, cluster) if not endpoint: return ( - _('No cluster endpoint for %r %r') % (realm, cluster), + _('No cluster endpoint for %(realm)r %(cluster)r') + % {'realm': realm, 'cluster': cluster}, None, None, None) return ( None, diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index d75ddbc07a..d78dec0110 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -196,9 +196,10 @@ def get_socket(conf): raise sleep(0.1) if not sock: - raise Exception(_('Could not bind to %s:%s ' - 'after trying for %s seconds') % ( - bind_addr[0], bind_addr[1], bind_timeout)) + raise Exception(_('Could not bind to %(addr)s:%(port)s ' + 'after trying for %(timeout)s seconds') % { + 'addr': bind_addr[0], 'port': bind_addr[1], + 'timeout': bind_timeout}) # in my experience, sockets can hang around forever without keepalive sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) @@ -1100,7 +1101,7 @@ def make_env(env, method=None, path=None, agent='Swift', query_string=None, 'swift.trans_id', 'swift.authorize_override', 'swift.authorize', 'HTTP_X_USER_ID', 'HTTP_X_PROJECT_ID', 'HTTP_REFERER', 'swift.orig_req_method', 'swift.log_info', - 'swift.metadata.checked'): + 'swift.infocache', 'swift.metadata.checked'): if name in env: newenv[name] = env[name] if method: diff --git a/swift/container/server.py b/swift/container/server.py index 92bb595e8f..a77dadcd22 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -183,11 +183,12 @@ class ContainerController(BaseStorageServer): if len(account_hosts) != len(account_devices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. - self.logger.error(_('ERROR Account update failed: different ' - 'numbers of hosts and devices in request: ' - '"%s" vs "%s"') % - (req.headers.get('X-Account-Host', ''), - req.headers.get('X-Account-Device', ''))) + self.logger.error(_( + 'ERROR Account update failed: different ' + 'numbers of hosts and devices in request: ' + '"%(hosts)s" vs "%(devices)s"') % { + 'hosts': req.headers.get('X-Account-Host', ''), + 'devices': req.headers.get('X-Account-Device', '')}) return HTTPBadRequest(req=req) if account_partition: diff --git a/swift/container/sync.py b/swift/container/sync.py index 8fbfe9dba3..1c94d9a679 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -237,8 +237,9 @@ class ContainerSync(Daemon): if err.errno != errno.ENOENT: raise raise SystemExit( - _('Unable to load internal client from config: %r (%s)') % - (internal_client_conf_path, err)) + _('Unable to load internal client from config: ' + '%(conf)r (%(error)s)') + % {'conf': internal_client_conf_path, 'error': err}) def run_forever(self, *args, **kwargs): """ diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py index f50ca7030e..49deb36f67 100644 --- a/swift/obj/diskfile.py +++ b/swift/obj/diskfile.py @@ -378,8 +378,9 @@ def object_audit_location_generator(devices, mount_check=True, logger=None, base, policy = split_policy_string(dir_) except PolicyError as e: if logger: - logger.warning(_('Directory %r does not map ' - 'to a valid policy (%s)') % (dir_, e)) + logger.warning(_('Directory %(directory)r does not map ' + 'to a valid policy (%(error)s)') % { + 'directory': dir_, 'error': e}) continue datadir_path = os.path.join(devices, device, dir_) diff --git a/swift/obj/expirer.py b/swift/obj/expirer.py index 6ecfd24829..115920dd6d 100644 --- a/swift/obj/expirer.py +++ b/swift/obj/expirer.py @@ -77,15 +77,17 @@ class ObjectExpirer(Daemon): """ if final: elapsed = time() - self.report_first_time - self.logger.info(_('Pass completed in %ds; %d objects expired') % - (elapsed, self.report_objects)) + self.logger.info(_('Pass completed in %(time)ds; ' + '%(objects)d objects expired') % { + 'time': elapsed, 'objects': self.report_objects}) dump_recon_cache({'object_expiration_pass': elapsed, 'expired_last_pass': self.report_objects}, self.rcache, self.logger) elif time() - self.report_last_time >= self.report_interval: elapsed = time() - self.report_first_time - self.logger.info(_('Pass so far %ds; %d objects expired') % - (elapsed, self.report_objects)) + self.logger.info(_('Pass so far %(time)ds; ' + '%(objects)d objects expired') % { + 'time': elapsed, 'objects': self.report_objects}) self.report_last_time = time() def iter_cont_objs_to_expire(self): @@ -168,8 +170,10 @@ class ObjectExpirer(Daemon): self.logger.debug('Run begin') containers, objects = \ self.swift.get_account_info(self.expiring_objects_account) - self.logger.info(_('Pass beginning; %s possible containers; %s ' - 'possible objects') % (containers, objects)) + self.logger.info(_('Pass beginning; ' + '%(containers)s possible containers; ' + '%(objects)s possible objects') % { + 'containers': containers, 'objects': objects}) for container, obj in self.iter_cont_objs_to_expire(): containers_to_delete.add(container) @@ -295,5 +299,6 @@ class ObjectExpirer(Daemon): """ path = '/v1/' + urllib.parse.quote(actual_obj.lstrip('/')) self.swift.make_request('DELETE', path, - {'X-If-Delete-At': str(timestamp)}, + {'X-If-Delete-At': str(timestamp), + 'X-Timestamp': str(timestamp)}, (2, HTTP_PRECONDITION_FAILED)) diff --git a/swift/obj/reconstructor.py b/swift/obj/reconstructor.py index 2c9e2b4c82..2b54ab89d2 100644 --- a/swift/obj/reconstructor.py +++ b/swift/obj/reconstructor.py @@ -843,6 +843,9 @@ class ObjectReconstructor(Daemon): self.part_count += len(partitions) for partition in partitions: part_path = join(obj_path, partition) + if partition in ('auditor_status_ALL.json', + 'auditor_status_ZBF.json'): + continue if not (partition.isdigit() and os.path.isdir(part_path)): self.logger.warning( diff --git a/swift/obj/server.py b/swift/obj/server.py index 267b45cacc..7b025cdcf9 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -282,11 +282,12 @@ class ObjectController(BaseStorageServer): if len(conthosts) != len(contdevices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. - self.logger.error(_('ERROR Container update failed: different ' - 'numbers of hosts and devices in request: ' - '"%s" vs "%s"') % - (headers_in.get('X-Container-Host', ''), - headers_in.get('X-Container-Device', ''))) + self.logger.error(_( + 'ERROR Container update failed: different ' + 'numbers of hosts and devices in request: ' + '"%(hosts)s" vs "%(devices)s"') % { + 'hosts': headers_in.get('X-Container-Host', ''), + 'devices': headers_in.get('X-Container-Device', '')}) return if contpartition: diff --git a/swift/obj/updater.py b/swift/obj/updater.py index 9bf4ef19a3..743cf850dc 100644 --- a/swift/obj/updater.py +++ b/swift/obj/updater.py @@ -160,9 +160,9 @@ class ObjectUpdater(Daemon): try: base, policy = split_policy_string(asyncdir) except PolicyError as e: - self.logger.warning(_('Directory %r does not map ' - 'to a valid policy (%s)') % - (asyncdir, e)) + self.logger.warning(_('Directory %(directory)r does not map ' + 'to a valid policy (%(error)s)') % { + 'directory': asyncdir, 'error': e}) continue for prefix in self._listdir(async_pending): prefix_path = os.path.join(async_pending, prefix) diff --git a/swift/proxy/controllers/account.py b/swift/proxy/controllers/account.py index faf4ccdee6..2abb3d1f79 100644 --- a/swift/proxy/controllers/account.py +++ b/swift/proxy/controllers/account.py @@ -24,7 +24,8 @@ from swift.common.utils import public from swift.common.constraints import check_metadata from swift.common import constraints from swift.common.http import HTTP_NOT_FOUND, HTTP_GONE -from swift.proxy.controllers.base import Controller, clear_info_cache +from swift.proxy.controllers.base import Controller, clear_info_cache, \ + set_info_cache from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed from swift.common.request_helpers import get_sys_meta_prefix @@ -57,6 +58,9 @@ class AccountController(Controller): resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), constraints.MAX_ACCOUNT_NAME_LENGTH) + # Don't cache this. We know the account doesn't exist because + # the name is bad; we don't need to cache that because it's + # really cheap to recompute. return resp partition = self.app.account_ring.get_part(self.account_name) @@ -70,8 +74,28 @@ class AccountController(Controller): if resp.headers.get('X-Account-Status', '').lower() == 'deleted': resp.status = HTTP_GONE elif self.app.account_autocreate: + # This is kind of a lie; we pretend like the account is + # there, but it's not. We'll create it as soon as something + # tries to write to it, but we don't need databases on disk + # to tell us that nothing's there. + # + # We set a header so that certain consumers can tell it's a + # fake listing. The important one is the PUT of a container + # to an autocreate account; the proxy checks to see if the + # account exists before actually performing the PUT and + # creates the account if necessary. If we feed it a perfect + # lie, it'll just try to create the container without + # creating the account, and that'll fail. resp = account_listing_response(self.account_name, req, get_listing_content_type(req)) + resp.headers['X-Backend-Fake-Account-Listing'] = 'yes' + + # Cache this. We just made a request to a storage node and got + # up-to-date information for the account. + resp.headers['X-Backend-Recheck-Account-Existence'] = str( + self.app.recheck_account_existence) + set_info_cache(self.app, req.environ, self.account_name, None, resp) + if req.environ.get('swift_owner'): self.add_acls_from_sys_metadata(resp) else: diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index a88e7f6b83..c1a909dad5 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -32,6 +32,7 @@ import functools import inspect import itertools import operator +from copy import deepcopy from sys import exc_info from swift import gettext_ as _ @@ -51,7 +52,7 @@ from swift.common.header_key_dict import HeaderKeyDict from swift.common.http import is_informational, is_success, is_redirection, \ is_server_error, HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_MULTIPLE_CHOICES, \ HTTP_BAD_REQUEST, HTTP_NOT_FOUND, HTTP_SERVICE_UNAVAILABLE, \ - HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED, HTTP_CONTINUE + HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED, HTTP_CONTINUE, HTTP_GONE from swift.common.swob import Request, Response, Range, \ HTTPException, HTTPRequestedRangeNotSatisfiable, HTTPServiceUnavailable, \ status_map @@ -62,6 +63,10 @@ from swift.common.request_helpers import strip_sys_meta_prefix, \ from swift.common.storage_policy import POLICIES +DEFAULT_RECHECK_ACCOUNT_EXISTENCE = 60 # seconds +DEFAULT_RECHECK_CONTAINER_EXISTENCE = 60 # seconds + + def update_headers(response, headers): """ Helper function to update headers in the response. @@ -103,18 +108,6 @@ def delay_denial(func): return func -def get_account_memcache_key(account): - cache_key, env_key = _get_cache_key(account, None) - return cache_key - - -def get_container_memcache_key(account, container): - if not container: - raise ValueError("container not provided") - cache_key, env_key = _get_cache_key(account, container) - return cache_key - - def _prep_headers_to_info(headers, server_type): """ Helper method that iterates once over a dict of headers, @@ -141,7 +134,7 @@ def headers_to_account_info(headers, status_int=HTTP_OK): Construct a cacheable dict of account info based on response headers. """ headers, meta, sysmeta = _prep_headers_to_info(headers, 'account') - return { + account_info = { 'status': status_int, # 'container_count' anomaly: # Previous code sometimes expects an int sometimes a string @@ -151,8 +144,12 @@ def headers_to_account_info(headers, status_int=HTTP_OK): 'total_object_count': headers.get('x-account-object-count'), 'bytes': headers.get('x-account-bytes-used'), 'meta': meta, - 'sysmeta': sysmeta + 'sysmeta': sysmeta, } + if is_success(status_int): + account_info['account_really_exists'] = not config_true_value( + headers.get('x-backend-fake-account-listing')) + return account_info def headers_to_container_info(headers, status_int=HTTP_OK): @@ -175,7 +172,7 @@ def headers_to_container_info(headers, status_int=HTTP_OK): 'max_age': meta.get('access-control-max-age') }, 'meta': meta, - 'sysmeta': sysmeta + 'sysmeta': sysmeta, } @@ -287,8 +284,17 @@ def get_object_info(env, app, path=None, swift_source=None): split_path(path or env['PATH_INFO'], 4, 4, True) info = _get_object_info(app, env, account, container, obj, swift_source=swift_source) - if not info: + if info: + info = deepcopy(info) + else: info = headers_to_object_info({}, 0) + + for field in ('length',): + if info.get(field) is None: + info[field] = 0 + else: + info[field] = int(info[field]) + return info @@ -304,11 +310,55 @@ def get_container_info(env, app, swift_source=None): """ (version, account, container, unused) = \ split_path(env['PATH_INFO'], 3, 4, True) - info = get_info(app, env, account, container, ret_not_found=True, - swift_source=swift_source) + + # Check in environment cache and in memcache (in that order) + info = _get_info_from_caches(app, env, account, container) + if not info: + # Cache miss; go HEAD the container and populate the caches + env.setdefault('swift.infocache', {}) + # Before checking the container, make sure the account exists. + # + # If it is an autocreateable account, just assume it exists; don't + # HEAD the account, as a GET or HEAD response for an autocreateable + # account is successful whether the account actually has .db files + # on disk or not. + is_autocreate_account = account.startswith( + getattr(app, 'auto_create_account_prefix', '.')) + if not is_autocreate_account: + account_info = get_account_info(env, app, swift_source) + if not account_info or not is_success(account_info['status']): + return headers_to_container_info({}, 0) + + req = _prepare_pre_auth_info_request( + env, ("/%s/%s/%s" % (version, account, container)), + (swift_source or 'GET_CONTAINER_INFO')) + resp = req.get_response(app) + # Check in infocache to see if the proxy (or anyone else) already + # populated the cache for us. If they did, just use what's there. + # + # See similar comment in get_account_info() for justification. + info = _get_info_from_infocache(env, account, container) + if info is None: + info = set_info_cache(app, env, account, container, resp) + + if info: + info = deepcopy(info) # avoid mutating what's in swift.infocache + else: info = headers_to_container_info({}, 0) + + # Old data format in memcache immediately after a Swift upgrade; clean + # it up so consumers of get_container_info() aren't exposed to it. info.setdefault('storage_policy', '0') + if 'object_count' not in info and 'container_size' in info: + info['object_count'] = info.pop('container_size') + + for field in ('bytes', 'object_count'): + if info.get(field) is None: + info[field] = 0 + else: + info[field] = int(info[field]) + return info @@ -322,32 +372,71 @@ def get_account_info(env, app, swift_source=None): This call bypasses auth. Success does not imply that the request has authorization to the account. - :raises ValueError: when path can't be split(path, 2, 4) + :raises ValueError: when path doesn't contain an account """ (version, account, _junk, _junk) = \ split_path(env['PATH_INFO'], 2, 4, True) - info = get_info(app, env, account, ret_not_found=True, - swift_source=swift_source) + + # Check in environment cache and in memcache (in that order) + info = _get_info_from_caches(app, env, account) + + # Cache miss; go HEAD the account and populate the caches if not info: - info = headers_to_account_info({}, 0) - if info.get('container_count') is None: - info['container_count'] = 0 + env.setdefault('swift.infocache', {}) + req = _prepare_pre_auth_info_request( + env, "/%s/%s" % (version, account), + (swift_source or 'GET_ACCOUNT_INFO')) + resp = req.get_response(app) + # Check in infocache to see if the proxy (or anyone else) already + # populated the cache for us. If they did, just use what's there. + # + # The point of this is to avoid setting the value in memcached + # twice. Otherwise, we're needlessly sending requests across the + # network. + # + # If the info didn't make it into the cache, we'll compute it from + # the response and populate the cache ourselves. + # + # Note that this is taking "exists in infocache" to imply "exists in + # memcache". That's because we're trying to avoid superfluous + # network traffic, and checking in memcache prior to setting in + # memcache would defeat the purpose. + info = _get_info_from_infocache(env, account) + if info is None: + info = set_info_cache(app, env, account, None, resp) + + if info: + info = info.copy() # avoid mutating what's in swift.infocache else: - info['container_count'] = int(info['container_count']) + info = headers_to_account_info({}, 0) + + for field in ('container_count', 'bytes', 'total_object_count'): + if info.get(field) is None: + info[field] = 0 + else: + info[field] = int(info[field]) + return info -def _get_cache_key(account, container): +def get_cache_key(account, container=None, obj=None): """ - Get the keys for both memcache (cache_key) and env (env_key) - where info about accounts and containers is cached + Get the keys for both memcache and env['swift.infocache'] (cache_key) + where info about accounts, containers, and objects is cached - :param account: The name of the account + :param account: The name of the account :param container: The name of the container (or None if account) - :returns: a tuple of (cache_key, env_key) + :param obj: The name of the object (or None if account or container) + :returns: a string cache_key """ - if container: + if obj: + if not (account and container): + raise ValueError('Object cache key requires account and container') + cache_key = 'object/%s/%s/%s' % (account, container, obj) + elif container: + if not account: + raise ValueError('Container cache key requires account') cache_key = 'container/%s/%s' % (account, container) else: cache_key = 'account/%s' % account @@ -355,58 +444,44 @@ def _get_cache_key(account, container): # This allows caching both account and container and ensures that when we # copy this env to form a new request, it won't accidentally reuse the # old container or account info - env_key = 'swift.%s' % cache_key - return cache_key, env_key + return cache_key -def get_object_env_key(account, container, obj): - """ - Get the keys for env (env_key) where info about object is cached - - :param account: The name of the account - :param container: The name of the container - :param obj: The name of the object - :returns: a string env_key - """ - env_key = 'swift.object/%s/%s/%s' % (account, - container, obj) - return env_key - - -def _set_info_cache(app, env, account, container, resp): +def set_info_cache(app, env, account, container, resp): """ Cache info in both memcache and env. - Caching is used to avoid unnecessary calls to account & container servers. - This is a private function that is being called by GETorHEAD_base and - by clear_info_cache. - Any attempt to GET or HEAD from the container/account server should use - the GETorHEAD_base interface which would than set the cache. - :param app: the application object :param account: the unquoted account name :param container: the unquoted container name or None - :param resp: the response received or None if info cache should be cleared - """ + :param resp: the response received or None if info cache should be cleared - if container: - cache_time = app.recheck_container_existence - else: - cache_time = app.recheck_account_existence - cache_key, env_key = _get_cache_key(account, container) + :returns: the info that was placed into the cache, or None if the + request status was not in (404, 410, 2xx). + """ + infocache = env.setdefault('swift.infocache', {}) + + cache_time = None + if container and resp: + cache_time = int(resp.headers.get( + 'X-Backend-Recheck-Container-Existence', + DEFAULT_RECHECK_CONTAINER_EXISTENCE)) + elif resp: + cache_time = int(resp.headers.get( + 'X-Backend-Recheck-Account-Existence', + DEFAULT_RECHECK_ACCOUNT_EXISTENCE)) + cache_key = get_cache_key(account, container) if resp: - if resp.status_int == HTTP_NOT_FOUND: + if resp.status_int in (HTTP_NOT_FOUND, HTTP_GONE): cache_time *= 0.1 elif not is_success(resp.status_int): cache_time = None - else: - cache_time = None # Next actually set both memcache and the env cache memcache = getattr(app, 'memcache', None) or env.get('swift.cache') if not cache_time: - env.pop(env_key, None) + infocache.pop(cache_key, None) if memcache: memcache.delete(cache_key) return @@ -417,35 +492,35 @@ def _set_info_cache(app, env, account, container, resp): info = headers_to_account_info(resp.headers, resp.status_int) if memcache: memcache.set(cache_key, info, time=cache_time) - env[env_key] = info + infocache[cache_key] = info + return info -def _set_object_info_cache(app, env, account, container, obj, resp): +def set_object_info_cache(app, env, account, container, obj, resp): """ - Cache object info env. Do not cache object information in - memcache. This is an intentional omission as it would lead - to cache pressure. This is a per-request cache. - - Caching is used to avoid unnecessary calls to object servers. - This is a private function that is being called by GETorHEAD_base. - Any attempt to GET or HEAD from the object server should use - the GETorHEAD_base interface which would then set the cache. + Cache object info in the WSGI environment, but not in memcache. Caching + in memcache would lead to cache pressure and mass evictions due to the + large number of objects in a typical Swift cluster. This is a + per-request cache only. :param app: the application object :param account: the unquoted account name - :param container: the unquoted container name or None - :param object: the unquoted object name or None - :param resp: the response received or None if info cache should be cleared + :param container: the unquoted container name + :param object: the unquoted object name + :param resp: a GET or HEAD response received from an object server, or + None if info cache should be cleared + :returns: the object info """ - env_key = get_object_env_key(account, container, obj) + cache_key = get_cache_key(account, container, obj) - if not resp: - env.pop(env_key, None) + if 'swift.infocache' in env and not resp: + env['swift.infocache'].pop(cache_key, None) return info = headers_to_object_info(resp.headers, resp.status_int) - env[env_key] = info + env.setdefault('swift.infocache', {})[cache_key] = info + return info def clear_info_cache(app, env, account, container=None): @@ -453,26 +528,43 @@ def clear_info_cache(app, env, account, container=None): Clear the cached info in both memcache and env :param app: the application object + :param env: the WSGI environment :param account: the account name :param container: the containr name or None if setting info for containers """ - _set_info_cache(app, env, account, container, None) + set_info_cache(app, env, account, container, None) -def _get_info_cache(app, env, account, container=None): +def _get_info_from_infocache(env, account, container=None): """ - Get the cached info from env or memcache (if used) in that order - Used for both account and container info - A private function used by get_info + Get cached account or container information from request-environment + cache (swift.infocache). + + :param env: the environment used by the current request + :param account: the account name + :param container: the container name + + :returns: a dictionary of cached info on cache hit, None on miss + """ + cache_key = get_cache_key(account, container) + if 'swift.infocache' in env and cache_key in env['swift.infocache']: + return env['swift.infocache'][cache_key] + return None + + +def _get_info_from_memcache(app, env, account, container=None): + """ + Get cached account or container information from memcache :param app: the application object :param env: the environment used by the current request - :returns: the cached info or None if not cached - """ + :param account: the account name + :param container: the container name - cache_key, env_key = _get_cache_key(account, container) - if env_key in env: - return env[env_key] + :returns: a dictionary of cached info on cache hit, None on miss. Also + returns None if memcache is not in use. + """ + cache_key = get_cache_key(account, container) memcache = getattr(app, 'memcache', None) or env.get('swift.cache') if memcache: info = memcache.get(cache_key) @@ -480,15 +572,31 @@ def _get_info_cache(app, env, account, container=None): for key in info: if isinstance(info[key], six.text_type): info[key] = info[key].encode("utf-8") - if isinstance(info[key], dict): + elif isinstance(info[key], dict): for subkey, value in info[key].items(): if isinstance(value, six.text_type): info[key][subkey] = value.encode("utf-8") - env[env_key] = info + env.setdefault('swift.infocache', {})[cache_key] = info return info return None +def _get_info_from_caches(app, env, account, container=None): + """ + Get the cached info from env or memcache (if used) in that order. + Used for both account and container info. + + :param app: the application object + :param env: the environment used by the current request + :returns: the cached info or None if not cached + """ + + info = _get_info_from_infocache(env, account, container) + if info is None: + info = _get_info_from_memcache(app, env, account, container) + return info + + def _prepare_pre_auth_info_request(env, path, swift_source): """ Prepares a pre authed request to obtain info using a HEAD. @@ -504,14 +612,18 @@ def _prepare_pre_auth_info_request(env, path, swift_source): # This is a sub request for container metadata- drop the Origin header from # the request so the it is not treated as a CORS request. newenv.pop('HTTP_ORIGIN', None) + + # ACLs are only shown to account owners, so let's make sure this request + # looks like it came from the account owner. + newenv['swift_owner'] = True + # Note that Request.blank expects quoted path return Request.blank(quote(path), environ=newenv) -def get_info(app, env, account, container=None, ret_not_found=False, - swift_source=None): +def get_info(app, env, account, container=None, swift_source=None): """ - Get the info about accounts or containers + Get info about accounts or containers Note: This call bypasses auth. Success does not imply that the request has authorization to the info. @@ -520,37 +632,25 @@ def get_info(app, env, account, container=None, ret_not_found=False, :param env: the environment used by the current request :param account: The unquoted name of the account :param container: The unquoted name of the container (or None if account) - :returns: the cached info or None if cannot be retrieved + :param swift_source: swift source logged for any subrequests made while + retrieving the account or container info + :returns: information about the specified entity in a dictionary. See + get_account_info and get_container_info for details on what's in the + dictionary. """ - info = _get_info_cache(app, env, account, container) - if info: - if ret_not_found or is_success(info['status']): - return info - return None - # Not in cache, let's try the account servers - path = '/v1/%s' % account - if container: - # Stop and check if we have an account? - if not get_info(app, env, account) and not account.startswith( - getattr(app, 'auto_create_account_prefix', '.')): - return None - path += '/' + container + env.setdefault('swift.infocache', {}) - req = _prepare_pre_auth_info_request( - env, path, (swift_source or 'GET_INFO')) - # Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in - # the environment under environ[env_key] and in memcache. We will - # pick the one from environ[env_key] and use it to set the caller env - resp = req.get_response(app) - cache_key, env_key = _get_cache_key(account, container) - try: - info = resp.environ[env_key] - env[env_key] = info - if ret_not_found or is_success(info['status']): - return info - except (KeyError, AttributeError): - pass - return None + if container: + path = '/v1/%s/%s' % (account, container) + path_env = env.copy() + path_env['PATH_INFO'] = path + return get_container_info(path_env, app, swift_source=swift_source) + else: + # account info + path = '/v1/%s' % (account,) + path_env = env.copy() + path_env['PATH_INFO'] = path + return get_account_info(path_env, app, swift_source=swift_source) def _get_object_info(app, env, account, container, obj, swift_source=None): @@ -567,24 +667,22 @@ def _get_object_info(app, env, account, container, obj, swift_source=None): :param obj: The unquoted name of the object :returns: the cached info or None if cannot be retrieved """ - env_key = get_object_env_key(account, container, obj) - info = env.get(env_key) + cache_key = get_cache_key(account, container, obj) + info = env.get('swift.infocache', {}).get(cache_key) if info: return info - # Not in cached, let's try the object servers + # Not in cache, let's try the object servers path = '/v1/%s/%s/%s' % (account, container, obj) req = _prepare_pre_auth_info_request(env, path, swift_source) - # Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in - # the environment under environ[env_key]. We will - # pick the one from environ[env_key] and use it to set the caller env resp = req.get_response(app) - try: - info = resp.environ[env_key] - env[env_key] = info - return info - except (KeyError, AttributeError): - pass - return None + # Unlike get_account_info() and get_container_info(), we don't save + # things in memcache, so we can store the info without network traffic, + # *and* the proxy doesn't cache object info for us, so there's no chance + # that the object info would be in the environment. Thus, we just + # compute the object info based on the response and stash it in + # swift.infocache. + info = set_object_info_cache(app, env, account, container, obj, resp) + return info def close_swift_conn(src): @@ -683,21 +781,21 @@ class ResumingGetter(object): if begin is None: # this is a -50 range req (last 50 bytes of file) end -= num_bytes + if end == 0: + # we sent out exactly the first range's worth of bytes, so + # we're done with it + raise RangeAlreadyComplete() else: begin += num_bytes - if end and begin == end + 1: - # we sent out exactly the first range's worth of bytes, so - # we're done with it - raise RangeAlreadyComplete() - elif end and begin > end: - raise HTTPRequestedRangeNotSatisfiable() - elif end and begin: - req_range.ranges = [(begin, end)] + req_range.ranges[1:] - elif end: - req_range.ranges = [(None, end)] + req_range.ranges[1:] - else: - req_range.ranges = [(begin, None)] + req_range.ranges[1:] + if end is not None and begin == end + 1: + # we sent out exactly the first range's worth of bytes, so + # we're done with it + raise RangeAlreadyComplete() + if end is not None and (begin > end or end < 0): + raise HTTPRequestedRangeNotSatisfiable() + + req_range.ranges = [(begin, end)] + req_range.ranges[1:] self.backend_headers['Range'] = str(req_range) else: self.backend_headers['Range'] = 'bytes=%d-' % num_bytes @@ -1355,8 +1453,14 @@ class Controller(object): env = getattr(req, 'environ', {}) else: env = {} - info = get_info(self.app, env, account) - if not info: + env.setdefault('swift.infocache', {}) + path_env = env.copy() + path_env['PATH_INFO'] = "/v1/%s" % (account,) + + info = get_account_info(path_env, self.app) + if (not info + or not is_success(info['status']) + or not info.get('account_really_exists', True)): return None, None, None if info.get('container_count') is None: container_count = 0 @@ -1383,8 +1487,11 @@ class Controller(object): env = getattr(req, 'environ', {}) else: env = {} - info = get_info(self.app, env, account, container) - if not info: + env.setdefault('swift.infocache', {}) + path_env = env.copy() + path_env['PATH_INFO'] = "/v1/%s/%s" % (account, container) + info = get_container_info(path_env, self.app) + if not info or not is_success(info.get('status')): info = headers_to_container_info({}, 0) info['partition'] = None info['nodes'] = None @@ -1672,17 +1779,7 @@ class Controller(object): req, handler.statuses, handler.reasons, handler.bodies, '%s %s' % (server_type, req.method), headers=handler.source_headers) - try: - (vrs, account, container) = req.split_path(2, 3) - _set_info_cache(self.app, req.environ, account, container, res) - except ValueError: - pass - try: - (vrs, account, container, obj) = req.split_path(4, 4, True) - _set_object_info_cache(self.app, req.environ, account, - container, obj, res) - except ValueError: - pass + # if a backend policy index is present in resp headers, translate it # here with the friendly policy name if 'X-Backend-Storage-Policy-Index' in res.headers and \ @@ -1697,6 +1794,7 @@ class Controller(object): 'Could not translate %s (%r) from %r to policy', 'X-Backend-Storage-Policy-Index', res.headers['X-Backend-Storage-Policy-Index'], path) + return res def is_origin_allowed(self, cors_info, origin): diff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py index 08a51f10d6..729f957aa9 100644 --- a/swift/proxy/controllers/container.py +++ b/swift/proxy/controllers/container.py @@ -22,7 +22,7 @@ from swift.common.constraints import check_metadata from swift.common import constraints from swift.common.http import HTTP_ACCEPTED, is_success from swift.proxy.controllers.base import Controller, delay_denial, \ - cors_validation, clear_info_cache + cors_validation, set_info_cache, clear_info_cache from swift.common.storage_policy import POLICIES from swift.common.swob import HTTPBadRequest, HTTPForbidden, \ HTTPNotFound @@ -85,11 +85,16 @@ class ContainerController(Controller): def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" - if not self.account_info(self.account_name, req)[1]: + ai = self.account_info(self.account_name, req) + if not ai[1]: if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: + # Don't cache this. It doesn't reflect the state of the + # container, just that the user can't access it. return aresp + # Don't cache this. The lack of account will be cached, and that + # is sufficient. return HTTPNotFound(request=req) part = self.app.container_ring.get_part( self.account_name, self.container_name) @@ -99,10 +104,18 @@ class ContainerController(Controller): resp = self.GETorHEAD_base( req, _('Container'), node_iter, part, req.swift_entity_path, concurrency) + # Cache this. We just made a request to a storage node and got + # up-to-date information for the container. + resp.headers['X-Backend-Recheck-Container-Existence'] = str( + self.app.recheck_container_existence) + set_info_cache(self.app, req.environ, self.account_name, + self.container_name, resp) if 'swift.authorize' in req.environ: req.acl = resp.headers.get('x-container-read') aresp = req.environ['swift.authorize'](req) if aresp: + # Don't cache this. It doesn't reflect the state of the + # container, just that the user can't access it. return aresp if not req.environ.get('swift_owner', False): for key in self.app.swift_owner_headers: diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 963bf34f0e..4993c90735 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -36,7 +36,8 @@ from swift.common.utils import cache_from_env, get_logger, \ from swift.common.constraints import check_utf8, valid_api_version from swift.proxy.controllers import AccountController, ContainerController, \ ObjectControllerRouter, InfoController -from swift.proxy.controllers.base import get_container_info, NodeIter +from swift.proxy.controllers.base import get_container_info, NodeIter, \ + DEFAULT_RECHECK_CONTAINER_EXISTENCE, DEFAULT_RECHECK_ACCOUNT_EXISTENCE from swift.common.swob import HTTPBadRequest, HTTPForbidden, \ HTTPMethodNotAllowed, HTTPNotFound, HTTPPreconditionFailed, \ HTTPServerError, HTTPException, Request, HTTPServiceUnavailable @@ -106,9 +107,11 @@ class Application(object): self.error_suppression_limit = \ int(conf.get('error_suppression_limit', 10)) self.recheck_container_existence = \ - int(conf.get('recheck_container_existence', 60)) + int(conf.get('recheck_container_existence', + DEFAULT_RECHECK_CONTAINER_EXISTENCE)) self.recheck_account_existence = \ - int(conf.get('recheck_account_existence', 60)) + int(conf.get('recheck_account_existence', + DEFAULT_RECHECK_ACCOUNT_EXISTENCE)) self.allow_account_management = \ config_true_value(conf.get('allow_account_management', 'no')) self.container_ring = container_ring or Ring(swift_dir, diff --git a/test/functional/tests.py b/test/functional/tests.py index 4beeb3aef0..aa1518571c 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -173,7 +173,9 @@ class TestAccount(Base): finally: self.env.account.conn.storage_url = was_url - def testPUT(self): + def testPUTError(self): + if load_constraint('allow_account_management'): + raise SkipTest("Allow account management is enabled") self.env.account.conn.make_request('PUT') self.assert_status([403, 405]) diff --git a/test/probe/brain.py b/test/probe/brain.py index ea5c2cc5ee..9f90ed8d8b 100644 --- a/test/probe/brain.py +++ b/test/probe/brain.py @@ -142,11 +142,16 @@ class BrainSplitter(object): """ put container with next storage policy """ - policy = next(self.policies) + if policy_index is not None: policy = POLICIES.get_by_index(int(policy_index)) if not policy: raise ValueError('Unknown policy with index %s' % policy) + elif not self.policy: + policy = next(self.policies) + else: + policy = self.policy + headers = {'X-Storage-Policy': policy.name} client.put_container(self.url, self.token, self.container_name, headers=headers) diff --git a/test/probe/test_container_merge_policy_index.py b/test/probe/test_container_merge_policy_index.py index 3472488f5f..829329a7eb 100644 --- a/test/probe/test_container_merge_policy_index.py +++ b/test/probe/test_container_merge_policy_index.py @@ -20,6 +20,7 @@ import unittest from nose import SkipTest +from six.moves.urllib.parse import urlparse from swift.common.manager import Manager from swift.common.internal_client import InternalClient from swift.common import utils, direct_client @@ -237,6 +238,14 @@ class TestContainerMergePolicyIndex(ReplProbeTest): orig_policy_index, node)) def test_reconcile_manifest(self): + info_url = "%s://%s/info" % (urlparse(self.url).scheme, + urlparse(self.url).netloc) + proxy_conn = client.http_connection(info_url) + cluster_info = client.get_capabilities(proxy_conn) + if 'slo' not in cluster_info: + raise SkipTest("SLO not enabled in proxy; " + "can't test manifest reconciliation") + # this test is not only testing a split brain scenario on # multiple policies with mis-placed objects - it even writes out # a static large object directly to the storage nodes while the @@ -278,18 +287,18 @@ class TestContainerMergePolicyIndex(ReplProbeTest): write_part(i) # write manifest - try: + with self.assertRaises(ClientException) as catcher: client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') - except ClientException as err: - # so as it works out, you can't really upload a multi-part - # manifest for objects that are currently misplaced - you have to - # wait until they're all available - which is about the same as - # some other failure that causes data to be unavailable to the - # proxy at the time of upload - self.assertEqual(err.http_status, 400) + + # so as it works out, you can't really upload a multi-part + # manifest for objects that are currently misplaced - you have to + # wait until they're all available - which is about the same as + # some other failure that causes data to be unavailable to the + # proxy at the time of upload + self.assertEqual(catcher.exception.http_status, 400) # but what the heck, we'll sneak one in just to see what happens... direct_manifest_name = self.object_name + '-direct-test' diff --git a/test/probe/test_object_expirer.py b/test/probe/test_object_expirer.py index 3f8f39deed..97351c746e 100644 --- a/test/probe/test_object_expirer.py +++ b/test/probe/test_object_expirer.py @@ -13,17 +13,18 @@ # limitations under the License. import random +import time import uuid import unittest from nose import SkipTest -from swift.common.internal_client import InternalClient +from swift.common.internal_client import InternalClient, UnexpectedResponse from swift.common.manager import Manager from swift.common.utils import Timestamp from test.probe.common import ReplProbeTest, ENABLED_POLICIES -from test.probe.test_container_merge_policy_index import BrainSplitter +from test.probe.brain import BrainSplitter from swiftclient import client @@ -31,9 +32,6 @@ from swiftclient import client class TestObjectExpirer(ReplProbeTest): def setUp(self): - if len(ENABLED_POLICIES) < 2: - raise SkipTest('Need more than one policy') - self.expirer = Manager(['object-expirer']) self.expirer.start() err = self.expirer.stop() @@ -53,6 +51,9 @@ class TestObjectExpirer(ReplProbeTest): self.object_name) def test_expirer_object_split_brain(self): + if len(ENABLED_POLICIES) < 2: + raise SkipTest('Need more than one policy') + old_policy = random.choice(ENABLED_POLICIES) wrong_policy = random.choice([p for p in ENABLED_POLICIES if p != old_policy]) @@ -126,5 +127,105 @@ class TestObjectExpirer(ReplProbeTest): self.assertTrue(Timestamp(metadata['x-backend-timestamp']) > create_timestamp) + def test_expirer_object_should_not_be_expired(self): + + # Current object-expirer checks the correctness via x-if-delete-at + # header that it can be deleted by expirer. If there are objects + # either which doesn't have x-delete-at header as metadata or which + # has different x-delete-at value from x-if-delete-at value, + # object-expirer's delete will fail as 412 PreconditionFailed. + # However, if some of the objects are in handoff nodes, the expirer + # can put the tombstone with the timestamp as same as x-delete-at and + # the object consistency will be resolved as the newer timestamp will + # be winner (in particular, overwritten case w/o x-delete-at). This + # test asserts such a situation that, at least, the overwriten object + # which have larger timestamp than the original expirered date should + # be safe. + + def put_object(headers): + # use internal client to PUT objects so that X-Timestamp in headers + # is effective + headers['Content-Length'] = '0' + path = self.client.make_path( + self.account, self.container_name, self.object_name) + try: + self.client.make_request('PUT', path, headers, (2,)) + except UnexpectedResponse as e: + self.fail( + 'Expected 201 for PUT object but got %s' % e.resp.status) + + obj_brain = BrainSplitter(self.url, self.token, self.container_name, + self.object_name, 'object', self.policy) + + # T(obj_created) < T(obj_deleted with x-delete-at) < T(obj_recreated) + # < T(expirer_executed) + # Recreated obj should be appeared in any split brain case + + obj_brain.put_container() + + # T(obj_deleted with x-delete-at) + # object-server accepts req only if X-Delete-At is later than 'now' + # so here, T(obj_created) < T(obj_deleted with x-delete-at) + now = time.time() + delete_at = int(now + 2.0) + recreate_at = delete_at + 1.0 + put_object(headers={'X-Delete-At': delete_at, + 'X-Timestamp': Timestamp(now).normal}) + + # some object servers stopped to make a situation that the + # object-expirer can put tombstone in the primary nodes. + obj_brain.stop_primary_half() + + # increment the X-Timestamp explicitly + # (will be T(obj_deleted with x-delete-at) < T(obj_recreated)) + put_object(headers={'X-Object-Meta-Expired': 'False', + 'X-Timestamp': Timestamp(recreate_at).normal}) + + # make sure auto-created containers get in the account listing + Manager(['container-updater']).once() + # sanity, the newer object is still there + try: + metadata = self.client.get_object_metadata( + self.account, self.container_name, self.object_name) + except UnexpectedResponse as e: + self.fail( + 'Expected 200 for HEAD object but got %s' % e.resp.status) + + self.assertIn('x-object-meta-expired', metadata) + + # some object servers recovered + obj_brain.start_primary_half() + + # sleep until after recreated_at + while time.time() <= recreate_at: + time.sleep(0.1) + # Now, expirer runs at the time after obj is recreated + self.expirer.once() + + # verify that original object was deleted by expirer + obj_brain.stop_handoff_half() + try: + metadata = self.client.get_object_metadata( + self.account, self.container_name, self.object_name, + acceptable_statuses=(4,)) + except UnexpectedResponse as e: + self.fail( + 'Expected 404 for HEAD object but got %s' % e.resp.status) + obj_brain.start_handoff_half() + + # and inconsistent state of objects is recovered by replicator + Manager(['object-replicator']).once() + + # check if you can get recreated object + try: + metadata = self.client.get_object_metadata( + self.account, self.container_name, self.object_name) + except UnexpectedResponse as e: + self.fail( + 'Expected 200 for HEAD object but got %s' % e.resp.status) + + self.assertIn('x-object-meta-expired', metadata) + + if __name__ == "__main__": unittest.main() diff --git a/test/unit/common/middleware/test_account_quotas.py b/test/unit/common/middleware/test_account_quotas.py index b443b4a28d..3ebbe37bd7 100644 --- a/test/unit/common/middleware/test_account_quotas.py +++ b/test/unit/common/middleware/test_account_quotas.py @@ -18,9 +18,8 @@ from swift.common.swob import Request, wsgify, HTTPForbidden, \ from swift.common.middleware import account_quotas, copy -from swift.proxy.controllers.base import _get_cache_key, \ - headers_to_account_info, get_object_env_key, \ - headers_to_object_info +from swift.proxy.controllers.base import get_cache_key, \ + headers_to_account_info, headers_to_object_info class FakeCache(object): @@ -58,16 +57,18 @@ class FakeApp(object): return aresp(env, start_response) if env['REQUEST_METHOD'] == "HEAD" and \ env['PATH_INFO'] == '/v1/a/c2/o2': - env_key = get_object_env_key('a', 'c2', 'o2') - env[env_key] = headers_to_object_info(self.headers, 200) + cache_key = get_cache_key('a', 'c2', 'o2') + env.setdefault('swift.infocache', {})[cache_key] = \ + headers_to_object_info(self.headers, 200) start_response('200 OK', self.headers) elif env['REQUEST_METHOD'] == "HEAD" and \ env['PATH_INFO'] == '/v1/a/c2/o3': start_response('404 Not Found', []) else: # Cache the account_info (same as a real application) - cache_key, env_key = _get_cache_key('a', None) - env[env_key] = headers_to_account_info(self.headers, 200) + cache_key = get_cache_key('a') + env.setdefault('swift.infocache', {})[cache_key] = \ + headers_to_account_info(self.headers, 200) start_response('200 OK', self.headers) return [] diff --git a/test/unit/common/middleware/test_container_sync.py b/test/unit/common/middleware/test_container_sync.py index 61a4735f15..6d30eeb6b1 100644 --- a/test/unit/common/middleware/test_container_sync.py +++ b/test/unit/common/middleware/test_container_sync.py @@ -23,7 +23,7 @@ import mock from swift.common import swob from swift.common.middleware import container_sync -from swift.proxy.controllers.base import _get_cache_key +from swift.proxy.controllers.base import get_cache_key from swift.proxy.controllers.info import InfoController from test.unit import FakeLogger @@ -205,7 +205,8 @@ cluster_dfw1 = http://dfw1.host/v1/ def test_invalid_sig(self): req = swob.Request.blank( '/v1/a/c', headers={'x-container-sync-auth': 'US nonce sig'}) - req.environ[_get_cache_key('a', 'c')[1]] = {'sync_key': 'abc'} + infocache = req.environ.setdefault('swift.infocache', {}) + infocache[get_cache_key('a', 'c')] = {'sync_key': 'abc'} resp = req.get_response(self.sync) self.assertEqual(resp.status, '401 Unauthorized') self.assertEqual( @@ -224,7 +225,8 @@ cluster_dfw1 = http://dfw1.host/v1/ req = swob.Request.blank('/v1/a/c', headers={ 'x-container-sync-auth': 'US nonce ' + sig, 'x-backend-inbound-x-timestamp': ts}) - req.environ[_get_cache_key('a', 'c')[1]] = {'sync_key': 'abc'} + infocache = req.environ.setdefault('swift.infocache', {}) + infocache[get_cache_key('a', 'c')] = {'sync_key': 'abc'} resp = req.get_response(self.sync) self.assertEqual(resp.status, '200 OK') self.assertEqual(resp.body, 'Response to Authorized Request') @@ -238,7 +240,8 @@ cluster_dfw1 = http://dfw1.host/v1/ self.sync.realms_conf.key2('US'), 'abc') req = swob.Request.blank( '/v1/a/c', headers={'x-container-sync-auth': 'US nonce ' + sig}) - req.environ[_get_cache_key('a', 'c')[1]] = {'sync_key': 'abc'} + infocache = req.environ.setdefault('swift.infocache', {}) + infocache[get_cache_key('a', 'c')] = {'sync_key': 'abc'} resp = req.get_response(self.sync) self.assertEqual(resp.status, '200 OK') self.assertEqual(resp.body, 'Response to Authorized Request') diff --git a/test/unit/common/middleware/test_copy.py b/test/unit/common/middleware/test_copy.py index 190d7c9084..254203e630 100644 --- a/test/unit/common/middleware/test_copy.py +++ b/test/unit/common/middleware/test_copy.py @@ -1073,6 +1073,7 @@ class TestServerSideCopyConfiguration(unittest.TestCase): @patch_policies(with_ec_default=True) class TestServerSideCopyMiddlewareWithEC(unittest.TestCase): container_info = { + 'status': 200, 'write_acl': None, 'read_acl': None, 'storage_policy': None, diff --git a/test/unit/common/middleware/test_formpost.py b/test/unit/common/middleware/test_formpost.py index 4e6f24826f..fabfe1931f 100644 --- a/test/unit/common/middleware/test_formpost.py +++ b/test/unit/common/middleware/test_formpost.py @@ -24,6 +24,7 @@ from six import BytesIO from swift.common.swob import Request, Response from swift.common.middleware import tempauth, formpost from swift.common.utils import split_path +from swift.proxy.controllers.base import get_cache_key class FakeApp(object): @@ -130,8 +131,9 @@ class TestFormPost(unittest.TestCase): meta[meta_name] = key _junk, account, _junk, _junk = split_path(path, 2, 4) - req.environ['swift.account/' + account] = self._fake_cache_env( - account, tempurl_keys) + req.environ.setdefault('swift.infocache', {}) + req.environ['swift.infocache'][get_cache_key(account)] = \ + self._fake_cache_env(account, tempurl_keys) return req def _fake_cache_env(self, account, tempurl_keys=()): @@ -221,6 +223,7 @@ class TestFormPost(unittest.TestCase): 'SERVER_NAME': '172.16.83.128', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', + 'swift.infocache': {}, 'wsgi.errors': wsgi_errors, 'wsgi.multiprocess': False, 'wsgi.multithread': True, @@ -247,8 +250,8 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() - 10), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -351,9 +354,11 @@ class TestFormPost(unittest.TestCase): 'SERVER_NAME': '172.16.83.128', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', - 'swift.account/AUTH_test': self._fake_cache_env( - 'AUTH_test', [key]), - 'swift.container/AUTH_test/container': {'meta': {}}, + 'swift.infocache': { + get_cache_key('AUTH_test'): self._fake_cache_env( + 'AUTH_test', [key]), + get_cache_key('AUTH_test', 'container'): { + 'meta': {}}}, 'wsgi.errors': wsgi_errors, 'wsgi.input': wsgi_input, 'wsgi.multiprocess': False, @@ -467,9 +472,11 @@ class TestFormPost(unittest.TestCase): 'SERVER_NAME': '172.16.83.128', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', - 'swift.account/AUTH_test': self._fake_cache_env( - 'AUTH_test', [key]), - 'swift.container/AUTH_test/container': {'meta': {}}, + 'swift.infocache': { + get_cache_key('AUTH_test'): self._fake_cache_env( + 'AUTH_test', [key]), + get_cache_key('AUTH_test', 'container'): { + 'meta': {}}}, 'wsgi.errors': wsgi_errors, 'wsgi.input': wsgi_input, 'wsgi.multiprocess': False, @@ -586,9 +593,11 @@ class TestFormPost(unittest.TestCase): 'SERVER_NAME': '172.16.83.128', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', - 'swift.account/AUTH_test': self._fake_cache_env( - 'AUTH_test', [key]), - 'swift.container/AUTH_test/container': {'meta': {}}, + 'swift.infocache': { + get_cache_key('AUTH_test'): self._fake_cache_env( + 'AUTH_test', [key]), + get_cache_key('AUTH_test', 'container'): { + 'meta': {}}}, 'wsgi.errors': wsgi_errors, 'wsgi.input': wsgi_input, 'wsgi.multiprocess': False, @@ -701,9 +710,11 @@ class TestFormPost(unittest.TestCase): 'SERVER_NAME': '172.16.83.128', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', - 'swift.account/AUTH_test': self._fake_cache_env( - 'AUTH_test', [key]), - 'swift.container/AUTH_test/container': {'meta': {}}, + 'swift.infocache': { + get_cache_key('AUTH_test'): self._fake_cache_env( + 'AUTH_test', [key]), + get_cache_key('AUTH_test', 'container'): { + 'meta': {}}}, 'wsgi.errors': wsgi_errors, 'wsgi.input': wsgi_input, 'wsgi.multiprocess': False, @@ -747,9 +758,10 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', 'http://brim.net', 5, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'XX' + b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -783,9 +795,10 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', 'http://brim.net', 5, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -814,9 +827,10 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', 'http://brim.net', 1024, 1, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -855,9 +869,10 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) env['QUERY_STRING'] = 'this=should¬=get&passed' env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp( iter([('201 Created', {}, ''), ('201 Created', {}, '')]), @@ -890,9 +905,10 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', 'http://brim.net', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp(iter([('404 Not Found', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -976,9 +992,10 @@ class TestFormPost(unittest.TestCase): if six.PY3: wsgi_input = wsgi_input.encode('utf-8') env['wsgi.input'] = BytesIO(wsgi_input) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1047,9 +1064,10 @@ class TestFormPost(unittest.TestCase): if six.PY3: wsgi_input = wsgi_input.encode('utf-8') env['wsgi.input'] = BytesIO(wsgi_input) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1087,9 +1105,10 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', 'http://redirect', 1024, 10, int(time() + 86400), key, user_agent=False) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1108,9 +1127,10 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', 'http://redirect', 1024, 10, int(time() + 86400), key, user_agent=False) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} env['HTTP_ORIGIN'] = 'http://localhost:5000' self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', @@ -1137,9 +1157,10 @@ class TestFormPost(unittest.TestCase): int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) # Stick it in X-Account-Meta-Temp-URL-Key-2 and make sure we get it - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', ['bert', key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1173,9 +1194,11 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', 'http://redirect', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env('AUTH_test') + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test')) # Stick it in X-Container-Meta-Temp-URL-Key-2 and ensure we get it - env['swift.container/AUTH_test/container'] = {'meta': meta} + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': meta} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1199,9 +1222,10 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', 'http://redirect', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1237,9 +1261,10 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', 'http://redirect?one=two', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1275,9 +1300,10 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1312,8 +1338,8 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() - 10), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1346,8 +1372,8 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) # Change key to invalidate sig - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key + ' is bogus now']) + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key + ' is bogus now'])) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1379,8 +1405,8 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'XX' + b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1412,8 +1438,8 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v2/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1445,8 +1471,8 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '//AUTH_test/container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1478,8 +1504,8 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1//container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1511,8 +1537,8 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_tst/container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) self.app = FakeApp(iter([ ('200 Ok', {'x-account-meta-temp-url-key': 'def'}, ''), ('201 Created', {}, ''), @@ -1546,8 +1572,8 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1584,8 +1610,8 @@ class TestFormPost(unittest.TestCase): body[i] = 'badvalue' break env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1625,9 +1651,10 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) wsgi_input = b'\r\n'.join(x_delete_body_part + body) env['wsgi.input'] = BytesIO(wsgi_input) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1668,8 +1695,8 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) wsgi_input = b'\r\n'.join(x_delete_body_part + body) env['wsgi.input'] = BytesIO(wsgi_input) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1703,9 +1730,10 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) wsgi_input = b'\r\n'.join(x_delete_body_part + body) env['wsgi.input'] = BytesIO(wsgi_input) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) - env['swift.container/AUTH_test/container'] = {'meta': {}} + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) + env['swift.infocache'][get_cache_key( + 'AUTH_test', 'container')] = {'meta': {}} self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) @@ -1746,8 +1774,8 @@ class TestFormPost(unittest.TestCase): '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) wsgi_input = b'\r\n'.join(x_delete_body_part + body) env['wsgi.input'] = BytesIO(wsgi_input) - env['swift.account/AUTH_test'] = self._fake_cache_env( - 'AUTH_test', [key]) + env['swift.infocache'][get_cache_key('AUTH_test')] = ( + self._fake_cache_env('AUTH_test', [key])) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) diff --git a/test/unit/common/middleware/test_keystoneauth.py b/test/unit/common/middleware/test_keystoneauth.py index 08aa86b3de..96d0ed4902 100644 --- a/test/unit/common/middleware/test_keystoneauth.py +++ b/test/unit/common/middleware/test_keystoneauth.py @@ -19,7 +19,7 @@ from swift.common.middleware import keystoneauth from swift.common.swob import Request, Response from swift.common.http import HTTP_FORBIDDEN from swift.common.utils import split_path -from swift.proxy.controllers.base import _get_cache_key +from swift.proxy.controllers.base import get_cache_key from test.unit import FakeLogger UNKNOWN_ID = keystoneauth.UNKNOWN_ID @@ -251,8 +251,8 @@ class SwiftAuth(unittest.TestCase): account = get_account_for_tenant(self.test_auth, proj_id) path = '/v1/' + account # fake cached account info - _, info_key = _get_cache_key(account, None) - env = {info_key: {'status': 0, 'sysmeta': {}}, + info_key = get_cache_key(account) + env = {'swift.infocache': {info_key: {'status': 0, 'sysmeta': {}}}, 'keystone.token_info': _fake_token_info(version='3')} req = Request.blank(path, environ=env, headers=headers) req.method = 'POST' @@ -280,8 +280,8 @@ class SwiftAuth(unittest.TestCase): account = get_account_for_tenant(self.test_auth, proj_id) path = '/v1/' + account # fake cached account info - _, info_key = _get_cache_key(account, None) - env = {info_key: {'status': 0, 'sysmeta': {}}, + info_key = get_cache_key(account) + env = {'swift.infocache': {info_key: {'status': 0, 'sysmeta': {}}}, 'keystone.token_info': _fake_token_info(version='3')} req = Request.blank(path, environ=env, headers=headers) req.method = 'POST' @@ -301,9 +301,9 @@ class SwiftAuth(unittest.TestCase): headers = get_identity_headers(tenant_id=proj_id, role='admin') account = get_account_for_tenant(self.test_auth, proj_id) path = '/v1/' + account - _, info_key = _get_cache_key(account, None) + info_key = get_cache_key(account) # v2 token - env = {info_key: {'status': 0, 'sysmeta': {}}, + env = {'swift.infocache': {info_key: {'status': 0, 'sysmeta': {}}}, 'keystone.token_info': _fake_token_info(version='2')} req = Request.blank(path, environ=env, headers=headers) req.method = 'POST' @@ -323,9 +323,9 @@ class SwiftAuth(unittest.TestCase): role='reselleradmin') account = get_account_for_tenant(self.test_auth, proj_id) path = '/v1/' + account - _, info_key = _get_cache_key(account, None) + info_key = get_cache_key(account) # v2 token - env = {info_key: {'status': 0, 'sysmeta': {}}, + env = {'swift.infocache': {info_key: {'status': 0, 'sysmeta': {}}}, 'keystone.token_info': _fake_token_info(version='2')} req = Request.blank(path, environ=env, headers=headers) req.method = 'POST' @@ -381,8 +381,8 @@ class ServiceTokenFunctionality(unittest.TestCase): role=user_role, service_role=service_role) (version, account, _junk, _junk) = split_path(path, 2, 4, True) - _, info_key = _get_cache_key(account, None) - env = {info_key: {'status': 0, 'sysmeta': {}}, + info_key = get_cache_key(account) + env = {'swift.infocache': {info_key: {'status': 0, 'sysmeta': {}}}, 'keystone.token_info': _fake_token_info(version='2')} if environ: env.update(environ) @@ -595,9 +595,10 @@ class TestAuthorize(BaseTestAuthorize): if not path: path = '/v1/%s/c' % account # fake cached account info - _, info_key = _get_cache_key(account, None) - default_env = {'REMOTE_USER': identity['HTTP_X_TENANT_ID'], - info_key: {'status': 200, 'sysmeta': {}}} + info_key = get_cache_key(account) + default_env = { + 'REMOTE_USER': identity['HTTP_X_TENANT_ID'], + 'swift.infocache': {info_key: {'status': 200, 'sysmeta': {}}}} default_env.update(identity) if env: default_env.update(env) @@ -984,9 +985,9 @@ class TestAuthorize(BaseTestAuthorize): def test_get_project_domain_id(self): sysmeta = {} info = {'sysmeta': sysmeta} - _, info_key = _get_cache_key('AUTH_1234', None) + info_key = get_cache_key('AUTH_1234') env = {'PATH_INFO': '/v1/AUTH_1234', - info_key: info} + 'swift.infocache': {info_key: info}} # account does not exist info['status'] = 404 @@ -1028,8 +1029,9 @@ class TestIsNameAllowedInACL(BaseTestAuthorize): # pretend account exists info = {'status': 200, 'sysmeta': sysmeta} - _, info_key = _get_cache_key(account, None) - req = Request.blank(path, environ={info_key: info}) + info_key = get_cache_key(account) + req = Request.blank(path, + environ={'swift.infocache': {info_key: info}}) if scoped == 'account': project_name = 'account_name' @@ -1214,8 +1216,8 @@ class TestSetProjectDomain(BaseTestAuthorize): if sysmeta_project_domain_id: sysmeta['project-domain-id'] = sysmeta_project_domain_id info = {'status': status, 'sysmeta': sysmeta} - _, info_key = _get_cache_key(account, None) - env = {info_key: info} + info_key = get_cache_key(account) + env = {'swift.infocache': {info_key: info}} # create fake env identity env_id = self._get_env_id(tenant_id=req_project_id, diff --git a/test/unit/common/middleware/test_quotas.py b/test/unit/common/middleware/test_quotas.py index f99b8df663..0bec5cad51 100644 --- a/test/unit/common/middleware/test_quotas.py +++ b/test/unit/common/middleware/test_quotas.py @@ -245,8 +245,9 @@ class ContainerQuotaCopyingTestCases(unittest.TestCase): 'status': 200, 'object_count': 1} req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'COPY', - 'swift.container/a/c': a_c_cache, - 'swift.container/a2/c': a2_c_cache}, + 'swift.infocache': { + 'container/a/c': a_c_cache, + 'container/a2/c': a2_c_cache}}, headers={'Destination': '/c/o', 'Destination-Account': 'a2'}) res = req.get_response(self.copy_filter) @@ -261,8 +262,9 @@ class ContainerQuotaCopyingTestCases(unittest.TestCase): 'status': 200, 'object_count': 1} req = Request.blank('/v1/a2/c/o', environ={'REQUEST_METHOD': 'PUT', - 'swift.container/a/c': a_c_cache, - 'swift.container/a2/c': a2_c_cache}, + 'swift.infocache': { + 'container/a/c': a_c_cache, + 'container/a2/c': a2_c_cache}}, headers={'X-Copy-From': '/c2/o2', 'X-Copy-From-Account': 'a'}) res = req.get_response(self.copy_filter) diff --git a/test/unit/common/middleware/test_ratelimit.py b/test/unit/common/middleware/test_ratelimit.py index 44136d2801..0fc6a61a11 100644 --- a/test/unit/common/middleware/test_ratelimit.py +++ b/test/unit/common/middleware/test_ratelimit.py @@ -21,7 +21,7 @@ from contextlib import contextmanager from test.unit import FakeLogger from swift.common.middleware import ratelimit -from swift.proxy.controllers.base import get_container_memcache_key, \ +from swift.proxy.controllers.base import get_cache_key, \ headers_to_container_info from swift.common.memcached import MemcacheConnectionError from swift.common.swob import Request @@ -185,7 +185,7 @@ class TestRateLimit(unittest.TestCase): conf_dict = {'account_ratelimit': current_rate, 'container_ratelimit_3': 200} fake_memcache = FakeMemcache() - fake_memcache.store[get_container_memcache_key('a', 'c')] = \ + fake_memcache.store[get_cache_key('a', 'c')] = \ {'object_count': '5'} the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache @@ -229,7 +229,7 @@ class TestRateLimit(unittest.TestCase): conf_dict = {'account_ratelimit': current_rate, 'container_ratelimit_3': 200} fake_memcache = FakeMemcache() - fake_memcache.store[get_container_memcache_key('a', 'c')] = \ + fake_memcache.store[get_cache_key('a', 'c')] = \ {'container_size': 5} the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache @@ -431,8 +431,8 @@ class TestRateLimit(unittest.TestCase): req.method = 'PUT' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].set( - get_container_memcache_key('a', 'c'), - {'container_size': 1}) + get_cache_key('a', 'c'), + {'object_count': 1}) time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping @@ -465,8 +465,8 @@ class TestRateLimit(unittest.TestCase): req.method = 'GET' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].set( - get_container_memcache_key('a', 'c'), - {'container_size': 1}) + get_cache_key('a', 'c'), + {'object_count': 1}) with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): diff --git a/test/unit/common/middleware/test_tempurl.py b/test/unit/common/middleware/test_tempurl.py index 0fc895f9e2..0d5ed07111 100644 --- a/test/unit/common/middleware/test_tempurl.py +++ b/test/unit/common/middleware/test_tempurl.py @@ -96,7 +96,8 @@ class TestTempURL(unittest.TestCase): if key: meta[meta_name] = key - environ['swift.account/' + account] = { + ic = environ.setdefault('swift.infocache', {}) + ic['account/' + account] = { 'status': 204, 'container_count': '0', 'total_object_count': '0', @@ -108,8 +109,8 @@ class TestTempURL(unittest.TestCase): meta_name = 'Temp-URL-key' + (("-%d" % (i + 1) if i else "")) meta[meta_name] = key - container_cache_key = 'swift.container/' + account + '/c' - environ.setdefault(container_cache_key, {'meta': meta}) + container_cache_key = 'container/' + account + '/c' + ic.setdefault(container_cache_key, {'meta': meta}) def test_passthrough(self): resp = self._make_request('/v1/a/c/o').get_response(self.tempurl) @@ -159,7 +160,8 @@ class TestTempURL(unittest.TestCase): self.assert_valid_sig(expires, path, [key1, key2], sig) def test_get_valid_container_keys(self): - environ = {} + ic = {} + environ = {'swift.infocache': ic} # Add two static container keys container_keys = ['me', 'other'] meta = {} @@ -167,7 +169,7 @@ class TestTempURL(unittest.TestCase): meta_name = 'Temp-URL-key' + (("-%d" % (idx + 1) if idx else "")) if key: meta[meta_name] = key - environ['swift.container/a/c'] = {'meta': meta} + ic['container/a/c'] = {'meta': meta} method = 'GET' expires = int(time() + 86400) diff --git a/test/unit/common/test_db_replicator.py b/test/unit/common/test_db_replicator.py index be17d42bf8..29d66df99d 100644 --- a/test/unit/common/test_db_replicator.py +++ b/test/unit/common/test_db_replicator.py @@ -73,7 +73,7 @@ class FakeRingWithSingleNode(object): class Ring(object): devs = [dict( id=1, weight=10.0, zone=1, ip='1.1.1.1', port=6200, device='sdb', - meta='', replication_ip='1.1.1.1', replication_port=6200 + meta='', replication_ip='1.1.1.1', replication_port=6200, region=1 )] def __init__(self, path, reload_time=15, ring_name=None): @@ -633,6 +633,9 @@ class TestDBReplicator(unittest.TestCase): def test_replicate_object_delete_because_not_shouldbehere(self): replicator = TestReplicator({}) + replicator.ring = FakeRingWithNodes().Ring('path') + replicator.brokerclass = FakeAccountBroker + replicator._repl_to_node = lambda *args: True replicator.delete_db = self.stub_delete_db replicator._replicate_object('0', '/path/to/file', 'node_id') self.assertEqual(['/path/to/file'], self.delete_db_calls) @@ -669,6 +672,30 @@ class TestDBReplicator(unittest.TestCase): [(('Found /path/to/file for /a%20c%20t/c%20o%20n when it should ' 'be on partition 0; will replicate out and remove.',), {})]) + def test_replicate_container_out_of_place_no_node(self): + replicator = TestReplicator({}, logger=unit.FakeLogger()) + replicator.ring = FakeRingWithSingleNode().Ring('path') + replicator._repl_to_node = lambda *args: True + + replicator.delete_db = self.stub_delete_db + # Correct node_id, wrong part + part = replicator.ring.get_part( + TEST_ACCOUNT_NAME, TEST_CONTAINER_NAME) + 1 + node_id = replicator.ring.get_part_nodes(part)[0]['id'] + replicator._replicate_object(str(part), '/path/to/file', node_id) + self.assertEqual(['/path/to/file'], self.delete_db_calls) + + self.delete_db_calls = [] + + # No nodes this time + replicator.ring.get_part_nodes = lambda *args: [] + replicator.delete_db = self.stub_delete_db + # Correct node_id, wrong part + part = replicator.ring.get_part( + TEST_ACCOUNT_NAME, TEST_CONTAINER_NAME) + 1 + replicator._replicate_object(str(part), '/path/to/file', node_id) + self.assertEqual([], self.delete_db_calls) + def test_replicate_object_different_region(self): db_replicator.ring = FakeRingWithNodes() replicator = TestReplicator({}) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index a69a092c0d..4c4e6fa819 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -2601,6 +2601,19 @@ cluster_dfw1 = http://dfw1.host/v1/ try: fallocate = utils.FallocateWrapper(noop=True) utils.os.fstatvfs = fstatvfs + + # Make sure setting noop, which disables fallocate, also stops the + # fallocate_reserve check. + # Set the fallocate_reserve to 99% and request an object that is + # about 50% the size. With fallocate_reserve off this will succeed. + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('99%') + self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(500)), 0) + + # Setting noop to False after the constructor allows us to use + # a noop fallocate syscall and still test fallocate_reserve. + fallocate.noop = False + # Want 1023 reserved, have 1024 * 1 free, so succeeds utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ utils.config_fallocate_value('1023') @@ -2618,67 +2631,68 @@ cluster_dfw1 = http://dfw1.host/v1/ utils.config_fallocate_value('1024') StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 - exc = None - try: + + with self.assertRaises(OSError) as catcher: fallocate(0, 1, 0, ctypes.c_uint64(0)) - except OSError as err: - exc = err - self.assertEqual(str(exc), - '[Errno 28] FALLOCATE_RESERVE fail 1024 <= 1024') - self.assertEqual(err.errno, errno.ENOSPC) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024' + % errno.ENOSPC) + self.assertEqual(catcher.exception.errno, errno.ENOSPC) + # Want 1024 reserved, have 512 * 2 free, so fails utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ utils.config_fallocate_value('1024') StatVFS.f_frsize = 512 StatVFS.f_bavail = 2 - exc = None - try: + with self.assertRaises(OSError) as catcher: fallocate(0, 1, 0, ctypes.c_uint64(0)) - except OSError as err: - exc = err - self.assertEqual(str(exc), - '[Errno 28] FALLOCATE_RESERVE fail 1024 <= 1024') - self.assertEqual(err.errno, errno.ENOSPC) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024' + % errno.ENOSPC) + self.assertEqual(catcher.exception.errno, errno.ENOSPC) + # Want 2048 reserved, have 1024 * 1 free, so fails utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ utils.config_fallocate_value('2048') StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 - exc = None - try: + with self.assertRaises(OSError) as catcher: fallocate(0, 1, 0, ctypes.c_uint64(0)) - except OSError as err: - exc = err - self.assertEqual(str(exc), - '[Errno 28] FALLOCATE_RESERVE fail 1024 <= 2048') - self.assertEqual(err.errno, errno.ENOSPC) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 1024 <= 2048' + % errno.ENOSPC) + self.assertEqual(catcher.exception.errno, errno.ENOSPC) + # Want 2048 reserved, have 512 * 2 free, so fails utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ utils.config_fallocate_value('2048') StatVFS.f_frsize = 512 StatVFS.f_bavail = 2 - exc = None - try: + with self.assertRaises(OSError) as catcher: fallocate(0, 1, 0, ctypes.c_uint64(0)) - except OSError as err: - exc = err - self.assertEqual(str(exc), - '[Errno 28] FALLOCATE_RESERVE fail 1024 <= 2048') - self.assertEqual(err.errno, errno.ENOSPC) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 1024 <= 2048' + % errno.ENOSPC) + self.assertEqual(catcher.exception.errno, errno.ENOSPC) + # Want 1023 reserved, have 1024 * 1 free, but file size is 1, so # fails utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ utils.config_fallocate_value('1023') StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 - exc = None - try: + with self.assertRaises(OSError) as catcher: fallocate(0, 1, 0, ctypes.c_uint64(1)) - except OSError as err: - exc = err - self.assertEqual(str(exc), - '[Errno 28] FALLOCATE_RESERVE fail 1023 <= 1023') - self.assertEqual(err.errno, errno.ENOSPC) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 1023 <= 1023' + % errno.ENOSPC) + self.assertEqual(catcher.exception.errno, errno.ENOSPC) + # Want 1022 reserved, have 1024 * 1 free, and file size is 1, so # succeeds utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ @@ -2686,6 +2700,7 @@ cluster_dfw1 = http://dfw1.host/v1/ StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(1)), 0) + # Want 1% reserved, have 100 bytes * 2/100 free, and file size is # 99, so succeeds utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ @@ -2694,6 +2709,7 @@ cluster_dfw1 = http://dfw1.host/v1/ StatVFS.f_bavail = 2 StatVFS.f_blocks = 100 self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(99)), 0) + # Want 2% reserved, have 50 bytes * 2/50 free, and file size is 49, # so succeeds utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ @@ -2702,6 +2718,7 @@ cluster_dfw1 = http://dfw1.host/v1/ StatVFS.f_bavail = 2 StatVFS.f_blocks = 50 self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(49)), 0) + # Want 100% reserved, have 100 * 100/100 free, and file size is 0, # so fails. utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ @@ -2709,15 +2726,14 @@ cluster_dfw1 = http://dfw1.host/v1/ StatVFS.f_frsize = 100 StatVFS.f_bavail = 100 StatVFS.f_blocks = 100 - exc = None - try: + with self.assertRaises(OSError) as catcher: fallocate(0, 1, 0, ctypes.c_uint64(0)) - except OSError as err: - exc = err - self.assertEqual(str(exc), - '[Errno 28] FALLOCATE_RESERVE fail 100.0 <= ' - '100.0') - self.assertEqual(err.errno, errno.ENOSPC) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 100.0 <= 100.0' + % errno.ENOSPC) + self.assertEqual(catcher.exception.errno, errno.ENOSPC) + # Want 1% reserved, have 100 * 2/100 free, and file size is 101, # so fails. utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ @@ -2725,29 +2741,28 @@ cluster_dfw1 = http://dfw1.host/v1/ StatVFS.f_frsize = 100 StatVFS.f_bavail = 2 StatVFS.f_blocks = 100 - exc = None - try: + with self.assertRaises(OSError) as catcher: fallocate(0, 1, 0, ctypes.c_uint64(101)) - except OSError as err: - exc = err - self.assertEqual(str(exc), - '[Errno 28] FALLOCATE_RESERVE fail 0.99 <= 1.0') - self.assertEqual(err.errno, errno.ENOSPC) - # Want 98% reserved, have 100 bytes * 99/100 free, and file size + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 0.99 <= 1.0' + % errno.ENOSPC) + self.assertEqual(catcher.exception.errno, errno.ENOSPC) + # is 100, so fails utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ utils.config_fallocate_value('98%') StatVFS.f_frsize = 100 StatVFS.f_bavail = 99 StatVFS.f_blocks = 100 - exc = None - try: + with self.assertRaises(OSError) as catcher: fallocate(0, 1, 0, ctypes.c_uint64(100)) - except OSError as err: - exc = err - self.assertEqual(str(exc), - '[Errno 28] FALLOCATE_RESERVE fail 98.0 <= 98.0') - self.assertEqual(err.errno, errno.ENOSPC) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 98.0 <= 98.0' + % errno.ENOSPC) + self.assertEqual(catcher.exception.errno, errno.ENOSPC) + # Want 2% reserved, have 1000 bytes * 21/1000 free, and file size # is 999, so succeeds. utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ @@ -2756,6 +2771,7 @@ cluster_dfw1 = http://dfw1.host/v1/ StatVFS.f_bavail = 21 StatVFS.f_blocks = 1000 self.assertEqual(fallocate(0, 1, 0, ctypes.c_uint64(999)), 0) + # Want 2% resereved, have 1000 bytes * 21/1000 free, and file size # is 1000, so fails. utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ @@ -2763,14 +2779,14 @@ cluster_dfw1 = http://dfw1.host/v1/ StatVFS.f_frsize = 1000 StatVFS.f_bavail = 21 StatVFS.f_blocks = 1000 - exc = None - try: + with self.assertRaises(OSError) as catcher: fallocate(0, 1, 0, ctypes.c_uint64(1000)) - except OSError as err: - exc = err - self.assertEqual(str(exc), - '[Errno 28] FALLOCATE_RESERVE fail 2.0 <= 2.0') - self.assertEqual(err.errno, errno.ENOSPC) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 2.0 <= 2.0' + % errno.ENOSPC) + self.assertEqual(catcher.exception.errno, errno.ENOSPC) + finally: utils.FALLOCATE_RESERVE = orig_FALLOCATE_RESERVE utils.os.fstatvfs = orig_fstatvfs diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py index 9c16daf805..cc33833714 100644 --- a/test/unit/common/test_wsgi.py +++ b/test/unit/common/test_wsgi.py @@ -839,6 +839,11 @@ class TestWSGI(unittest.TestCase): self.assertTrue('HTTP_REFERER' in newenv) self.assertEqual(newenv['HTTP_REFERER'], 'http://blah.example.com') + def test_make_env_keeps_infocache(self): + oldenv = {'swift.infocache': {}} + newenv = wsgi.make_env(oldenv) + self.assertIs(newenv.get('swift.infocache'), oldenv['swift.infocache']) + class TestServersPerPortStrategy(unittest.TestCase): def setUp(self): diff --git a/test/unit/obj/test_expirer.py b/test/unit/obj/test_expirer.py index 17fddf174d..02a04dda01 100644 --- a/test/unit/obj/test_expirer.py +++ b/test/unit/obj/test_expirer.py @@ -675,6 +675,8 @@ class TestObjectExpirer(TestCase): ts = '1234' x.delete_actual_object('/path/to/object', ts) self.assertEqual(got_env[0]['HTTP_X_IF_DELETE_AT'], ts) + self.assertEqual(got_env[0]['HTTP_X_TIMESTAMP'], + got_env[0]['HTTP_X_IF_DELETE_AT']) def test_delete_actual_object_nourlquoting(self): # delete_actual_object should not do its own url quoting because @@ -692,6 +694,8 @@ class TestObjectExpirer(TestCase): ts = '1234' x.delete_actual_object('/path/to/object name', ts) self.assertEqual(got_env[0]['HTTP_X_IF_DELETE_AT'], ts) + self.assertEqual(got_env[0]['HTTP_X_TIMESTAMP'], + got_env[0]['HTTP_X_IF_DELETE_AT']) self.assertEqual(got_env[0]['PATH_INFO'], '/v1/path/to/object name') def test_delete_actual_object_raises_404(self): diff --git a/test/unit/obj/test_reconstructor.py b/test/unit/obj/test_reconstructor.py index 13d29562a2..cf8474888b 100755 --- a/test/unit/obj/test_reconstructor.py +++ b/test/unit/obj/test_reconstructor.py @@ -823,8 +823,43 @@ class TestGlobalSetupObjectReconstructor(unittest.TestCase): self.assertFalse(os.path.exists(pol_1_part_1_path)) warnings = self.reconstructor.logger.get_lines_for_level('warning') self.assertEqual(1, len(warnings)) - self.assertTrue('Unexpected entity in data dir:' in warnings[0], - 'Warning not found in %s' % warnings) + self.assertIn('Unexpected entity in data dir:', warnings[0]) + + def test_ignores_status_file(self): + # Following fd86d5a, the auditor will leave status files on each device + # until an audit can complete. The reconstructor should ignore these + + @contextmanager + def status_files(*auditor_types): + status_paths = [os.path.join(self.objects_1, + 'auditor_status_%s.json' % typ) + for typ in auditor_types] + for status_path in status_paths: + self.assertFalse(os.path.exists(status_path)) # sanity check + with open(status_path, 'w'): + pass + self.assertTrue(os.path.isfile(status_path)) # sanity check + try: + yield status_paths + finally: + for status_path in status_paths: + try: + os.unlink(status_path) + except OSError as e: + if e.errno != 2: + raise + + # since our collect_parts job is a generator, that yields directly + # into build_jobs and then spawns it's safe to do the remove_files + # without making reconstructor startup slow + with status_files('ALL', 'ZBF') as status_paths: + self.reconstructor._reset_stats() + for part_info in self.reconstructor.collect_parts(): + self.assertNotIn(part_info['part_path'], status_paths) + warnings = self.reconstructor.logger.get_lines_for_level('warning') + self.assertEqual(0, len(warnings)) + for status_path in status_paths: + self.assertTrue(os.path.exists(status_path)) def _make_fake_ssync(self, ssync_calls): class _fake_ssync(object): diff --git a/test/unit/proxy/controllers/test_account.py b/test/unit/proxy/controllers/test_account.py index d3dd9cf504..86206f02a1 100644 --- a/test/unit/proxy/controllers/test_account.py +++ b/test/unit/proxy/controllers/test_account.py @@ -25,6 +25,7 @@ from test.unit import fake_http_connect, FakeRing, FakeMemcache from swift.common.storage_policy import StoragePolicy from swift.common.request_helpers import get_sys_meta_prefix import swift.proxy.controllers.base +from swift.proxy.controllers.base import get_account_info from test.unit import patch_policies @@ -68,9 +69,10 @@ class TestAccountController(unittest.TestCase): req = Request.blank('/v1/AUTH_bob', {'PATH_INFO': '/v1/AUTH_bob'}) resp = controller.HEAD(req) self.assertEqual(2, resp.status_int // 100) - self.assertTrue('swift.account/AUTH_bob' in resp.environ) - self.assertEqual(headers_to_account_info(resp.headers), - resp.environ['swift.account/AUTH_bob']) + self.assertIn('account/AUTH_bob', resp.environ['swift.infocache']) + self.assertEqual( + headers_to_account_info(resp.headers), + resp.environ['swift.infocache']['account/AUTH_bob']) def test_swift_owner(self): owner_headers = { @@ -225,13 +227,20 @@ class TestAccountController(unittest.TestCase): self.assertEqual(1, len(resp.headers)) # we always get Content-Type self.assertEqual(2, len(resp.environ)) - def test_memcache_key_impossible_cases(self): + def test_cache_key_impossible_cases(self): # For test coverage: verify that defensive coding does defend, in cases # that shouldn't arise naturally - self.assertRaises( - ValueError, - lambda: swift.proxy.controllers.base.get_container_memcache_key( - '/a', None)) + with self.assertRaises(ValueError): + # Container needs account + swift.proxy.controllers.base.get_cache_key(None, 'c') + + with self.assertRaises(ValueError): + # Object needs account + swift.proxy.controllers.base.get_cache_key(None, 'c', 'o') + + with self.assertRaises(ValueError): + # Object needs container + swift.proxy.controllers.base.get_cache_key('a', None, 'o') def test_stripping_swift_admin_headers(self): # Verify that a GET/HEAD which receives privileged headers from the @@ -376,5 +385,22 @@ class TestAccountController4Replicas(TestAccountController): self._assert_responses('POST', POST_TEST_CASES) +@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())]) +class TestGetAccountInfo(unittest.TestCase): + def setUp(self): + self.app = proxy_server.Application( + None, FakeMemcache(), + account_ring=FakeRing(), container_ring=FakeRing()) + + def test_get_deleted_account_410(self): + resp_headers = {'x-account-status': 'deleted'} + + req = Request.blank('/v1/a') + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(404, headers=resp_headers)): + info = get_account_info(req.environ, self.app) + self.assertEqual(410, info.get('status')) + + if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/controllers/test_base.py b/test/unit/proxy/controllers/test_base.py index 220c70fb97..55214f6d03 100644 --- a/test/unit/proxy/controllers/test_base.py +++ b/test/unit/proxy/controllers/test_base.py @@ -19,16 +19,14 @@ import unittest from mock import patch from swift.proxy.controllers.base import headers_to_container_info, \ headers_to_account_info, headers_to_object_info, get_container_info, \ - get_container_memcache_key, get_account_info, get_account_memcache_key, \ - get_object_env_key, get_info, get_object_info, \ - Controller, GetOrHeadHandler, _set_info_cache, _set_object_info_cache, \ - bytes_to_skip + get_cache_key, get_account_info, get_info, get_object_info, \ + Controller, GetOrHeadHandler, bytes_to_skip from swift.common.swob import Request, HTTPException, RESPONSE_REASONS from swift.common import exceptions from swift.common.utils import split_path from swift.common.header_key_dict import HeaderKeyDict from swift.common.http import is_success -from swift.common.storage_policy import StoragePolicy, POLICIES +from swift.common.storage_policy import StoragePolicy from test.unit import fake_http_connect, FakeRing, FakeMemcache from swift.proxy import server as proxy_server from swift.common.request_helpers import ( @@ -122,23 +120,14 @@ class FakeApp(object): def __init__(self, response_factory=None, statuses=None): self.responses = response_factory or \ DynamicResponseFactory(*statuses or []) - self.sources = [] + self.captured_envs = [] def __call__(self, environ, start_response): - self.sources.append(environ.get('swift.source')) + self.captured_envs.append(environ) response = self.responses.get_response(environ) reason = RESPONSE_REASONS[response.status_int][0] start_response('%d %s' % (response.status_int, reason), [(k, v) for k, v in response.headers.items()]) - # It's a bit strange, but the get_info cache stuff relies on the - # app setting some keys in the environment as it makes requests - # (in particular GETorHEAD_base) - so our fake does the same - _set_info_cache(self, environ, response.account, - response.container, response) - if response.obj: - _set_object_info_cache(self, environ, response.account, - response.container, response.obj, - response) return iter(response.body) @@ -160,87 +149,6 @@ class TestFuncs(unittest.TestCase): account_ring=FakeRing(), container_ring=FakeRing()) - def test_GETorHEAD_base(self): - base = Controller(self.app) - req = Request.blank('/v1/a/c/o/with/slashes') - ring = FakeRing() - nodes = list(ring.get_part_nodes(0)) + list(ring.get_more_nodes(0)) - with patch('swift.proxy.controllers.base.' - 'http_connect', fake_http_connect(200)): - resp = base.GETorHEAD_base(req, 'object', iter(nodes), 'part', - '/a/c/o/with/slashes') - self.assertTrue('swift.object/a/c/o/with/slashes' in resp.environ) - self.assertEqual( - resp.environ['swift.object/a/c/o/with/slashes']['status'], 200) - req = Request.blank('/v1/a/c/o') - with patch('swift.proxy.controllers.base.' - 'http_connect', fake_http_connect(200)): - resp = base.GETorHEAD_base(req, 'object', iter(nodes), 'part', - '/a/c/o') - self.assertTrue('swift.object/a/c/o' in resp.environ) - self.assertEqual(resp.environ['swift.object/a/c/o']['status'], 200) - req = Request.blank('/v1/a/c') - with patch('swift.proxy.controllers.base.' - 'http_connect', fake_http_connect(200)): - resp = base.GETorHEAD_base(req, 'container', iter(nodes), 'part', - '/a/c') - self.assertTrue('swift.container/a/c' in resp.environ) - self.assertEqual(resp.environ['swift.container/a/c']['status'], 200) - - req = Request.blank('/v1/a') - with patch('swift.proxy.controllers.base.' - 'http_connect', fake_http_connect(200)): - resp = base.GETorHEAD_base(req, 'account', iter(nodes), 'part', - '/a') - self.assertTrue('swift.account/a' in resp.environ) - self.assertEqual(resp.environ['swift.account/a']['status'], 200) - - # Run the above tests again, but this time with concurrent_reads - # turned on - policy = next(iter(POLICIES)) - concurrent_get_threads = policy.object_ring.replica_count - for concurrency_timeout in (0, 2): - self.app.concurrency_timeout = concurrency_timeout - req = Request.blank('/v1/a/c/o/with/slashes') - # NOTE: We are using slow_connect of fake_http_connect as using - # a concurrency of 0 when mocking the connection is a little too - # fast for eventlet. Network i/o will make this fine, but mocking - # it seems is too instantaneous. - with patch('swift.proxy.controllers.base.http_connect', - fake_http_connect(200, slow_connect=True)): - resp = base.GETorHEAD_base( - req, 'object', iter(nodes), 'part', '/a/c/o/with/slashes', - concurrency=concurrent_get_threads) - self.assertTrue('swift.object/a/c/o/with/slashes' in resp.environ) - self.assertEqual( - resp.environ['swift.object/a/c/o/with/slashes']['status'], 200) - req = Request.blank('/v1/a/c/o') - with patch('swift.proxy.controllers.base.http_connect', - fake_http_connect(200, slow_connect=True)): - resp = base.GETorHEAD_base( - req, 'object', iter(nodes), 'part', '/a/c/o', - concurrency=concurrent_get_threads) - self.assertTrue('swift.object/a/c/o' in resp.environ) - self.assertEqual(resp.environ['swift.object/a/c/o']['status'], 200) - req = Request.blank('/v1/a/c') - with patch('swift.proxy.controllers.base.http_connect', - fake_http_connect(200, slow_connect=True)): - resp = base.GETorHEAD_base( - req, 'container', iter(nodes), 'part', '/a/c', - concurrency=concurrent_get_threads) - self.assertTrue('swift.container/a/c' in resp.environ) - self.assertEqual(resp.environ['swift.container/a/c']['status'], - 200) - - req = Request.blank('/v1/a') - with patch('swift.proxy.controllers.base.http_connect', - fake_http_connect(200, slow_connect=True)): - resp = base.GETorHEAD_base( - req, 'account', iter(nodes), 'part', '/a', - concurrency=concurrent_get_threads) - self.assertTrue('swift.account/a' in resp.environ) - self.assertEqual(resp.environ['swift.account/a']['status'], 200) - def test_get_info(self): app = FakeApp() # Do a non cached call to account @@ -250,36 +158,44 @@ class TestFuncs(unittest.TestCase): self.assertEqual(info_a['status'], 200) self.assertEqual(info_a['bytes'], 6666) self.assertEqual(info_a['total_object_count'], 1000) - # Make sure the env cache is set - self.assertEqual(env.get('swift.account/a'), info_a) + # Make sure the app was called self.assertEqual(app.responses.stats['account'], 1) + # Make sure the return value matches get_account_info + account_info = get_account_info({'PATH_INFO': '/v1/a'}, app) + self.assertEqual(info_a, account_info) + # Do an env cached call to account + app.responses.stats['account'] = 0 + app.responses.stats['container'] = 0 + info_a = get_info(app, env, 'a') # Check that you got proper info self.assertEqual(info_a['status'], 200) self.assertEqual(info_a['bytes'], 6666) self.assertEqual(info_a['total_object_count'], 1000) - # Make sure the env cache is set - self.assertEqual(env.get('swift.account/a'), info_a) + # Make sure the app was NOT called AGAIN - self.assertEqual(app.responses.stats['account'], 1) + self.assertEqual(app.responses.stats['account'], 0) # This time do env cached call to account and non cached to container + app.responses.stats['account'] = 0 + app.responses.stats['container'] = 0 + info_c = get_info(app, env, 'a', 'c') # Check that you got proper info self.assertEqual(info_c['status'], 200) self.assertEqual(info_c['bytes'], 6666) self.assertEqual(info_c['object_count'], 1000) - # Make sure the env cache is set - self.assertEqual(env.get('swift.account/a'), info_a) - self.assertEqual(env.get('swift.container/a/c'), info_c) - # Make sure the app was called for container + # Make sure the app was called for container but not account + self.assertEqual(app.responses.stats['account'], 0) self.assertEqual(app.responses.stats['container'], 1) - # This time do a non cached call to account than non cached to + # This time do a non-cached call to account then non-cached to # container + app.responses.stats['account'] = 0 + app.responses.stats['container'] = 0 app = FakeApp() env = {} # abandon previous call to env info_c = get_info(app, env, 'a', 'c') @@ -287,81 +203,38 @@ class TestFuncs(unittest.TestCase): self.assertEqual(info_c['status'], 200) self.assertEqual(info_c['bytes'], 6666) self.assertEqual(info_c['object_count'], 1000) - # Make sure the env cache is set - self.assertEqual(env.get('swift.account/a'), info_a) - self.assertEqual(env.get('swift.container/a/c'), info_c) # check app calls both account and container self.assertEqual(app.responses.stats['account'], 1) self.assertEqual(app.responses.stats['container'], 1) - # This time do an env cached call to container while account is not + # This time do an env-cached call to container while account is not # cached - del(env['swift.account/a']) + app.responses.stats['account'] = 0 + app.responses.stats['container'] = 0 info_c = get_info(app, env, 'a', 'c') # Check that you got proper info self.assertEqual(info_a['status'], 200) self.assertEqual(info_c['bytes'], 6666) self.assertEqual(info_c['object_count'], 1000) - # Make sure the env cache is set and account still not cached - self.assertEqual(env.get('swift.container/a/c'), info_c) + # no additional calls were made - self.assertEqual(app.responses.stats['account'], 1) - self.assertEqual(app.responses.stats['container'], 1) - - # Do a non cached call to account not found with ret_not_found - app = FakeApp(statuses=(404,)) - env = {} - info_a = get_info(app, env, 'a', ret_not_found=True) - # Check that you got proper info - self.assertEqual(info_a['status'], 404) - self.assertEqual(info_a['bytes'], None) - self.assertEqual(info_a['total_object_count'], None) - # Make sure the env cache is set - self.assertEqual(env.get('swift.account/a'), info_a) - # and account was called - self.assertEqual(app.responses.stats['account'], 1) - - # Do a cached call to account not found with ret_not_found - info_a = get_info(app, env, 'a', ret_not_found=True) - # Check that you got proper info - self.assertEqual(info_a['status'], 404) - self.assertEqual(info_a['bytes'], None) - self.assertEqual(info_a['total_object_count'], None) - # Make sure the env cache is set - self.assertEqual(env.get('swift.account/a'), info_a) - # add account was NOT called AGAIN - self.assertEqual(app.responses.stats['account'], 1) - - # Do a non cached call to account not found without ret_not_found - app = FakeApp(statuses=(404,)) - env = {} - info_a = get_info(app, env, 'a') - # Check that you got proper info - self.assertEqual(info_a, None) - self.assertEqual(env['swift.account/a']['status'], 404) - # and account was called - self.assertEqual(app.responses.stats['account'], 1) - - # Do a cached call to account not found without ret_not_found - info_a = get_info(None, env, 'a') - # Check that you got proper info - self.assertEqual(info_a, None) - self.assertEqual(env['swift.account/a']['status'], 404) - # add account was NOT called AGAIN - self.assertEqual(app.responses.stats['account'], 1) + self.assertEqual(app.responses.stats['account'], 0) + self.assertEqual(app.responses.stats['container'], 0) def test_get_container_info_swift_source(self): app = FakeApp() req = Request.blank("/v1/a/c", environ={'swift.cache': FakeCache()}) get_container_info(req.environ, app, swift_source='MC') - self.assertEqual(app.sources, ['GET_INFO', 'MC']) + self.assertEqual([e['swift.source'] for e in app.captured_envs], + ['MC', 'MC']) def test_get_object_info_swift_source(self): app = FakeApp() req = Request.blank("/v1/a/c/o", environ={'swift.cache': FakeCache()}) get_object_info(req.environ, app, swift_source='LU') - self.assertEqual(app.sources, ['LU']) + self.assertEqual([e['swift.source'] for e in app.captured_envs], + ['LU']) def test_get_container_info_no_cache(self): req = Request.blank("/v1/AUTH_account/cont", @@ -379,7 +252,7 @@ class TestFuncs(unittest.TestCase): self.assertEqual(info['status'], 0) def test_get_container_info_no_auto_account(self): - responses = DynamicResponseFactory(404, 200) + responses = DynamicResponseFactory(200) app = FakeApp(responses) req = Request.blank("/v1/.system_account/cont") info = get_container_info(req.environ, app) @@ -401,11 +274,11 @@ class TestFuncs(unittest.TestCase): self.assertEqual(resp['versions'], "\xe1\xbd\x8a\x39") def test_get_container_info_env(self): - cache_key = get_container_memcache_key("account", "cont") - env_key = 'swift.%s' % cache_key - req = Request.blank("/v1/account/cont", - environ={env_key: {'bytes': 3867}, - 'swift.cache': FakeCache({})}) + cache_key = get_cache_key("account", "cont") + req = Request.blank( + "/v1/account/cont", + environ={'swift.infocache': {cache_key: {'bytes': 3867}}, + 'swift.cache': FakeCache({})}) resp = get_container_info(req.environ, 'xxx') self.assertEqual(resp['bytes'], 3867) @@ -413,7 +286,25 @@ class TestFuncs(unittest.TestCase): app = FakeApp() req = Request.blank("/v1/a", environ={'swift.cache': FakeCache()}) get_account_info(req.environ, app, swift_source='MC') - self.assertEqual(app.sources, ['MC']) + self.assertEqual([e['swift.source'] for e in app.captured_envs], + ['MC']) + + def test_get_account_info_swift_owner(self): + app = FakeApp() + req = Request.blank("/v1/a", environ={'swift.cache': FakeCache()}) + get_account_info(req.environ, app) + self.assertEqual([e['swift_owner'] for e in app.captured_envs], + [True]) + + def test_get_account_info_infocache(self): + app = FakeApp() + ic = {} + req = Request.blank("/v1/a", environ={'swift.cache': FakeCache(), + 'swift.infocache': ic}) + get_account_info(req.environ, app) + got_infocaches = [e['swift.infocache'] for e in app.captured_envs] + self.assertEqual(1, len(got_infocaches)) + self.assertIs(ic, got_infocaches[0]) def test_get_account_info_no_cache(self): app = FakeApp() @@ -424,7 +315,7 @@ class TestFuncs(unittest.TestCase): self.assertEqual(resp['total_object_count'], 1000) def test_get_account_info_cache(self): - # The original test that we prefer to preserve + # Works with fake apps that return ints in the headers cached = {'status': 404, 'bytes': 3333, 'total_object_count': 10} @@ -435,7 +326,8 @@ class TestFuncs(unittest.TestCase): self.assertEqual(resp['total_object_count'], 10) self.assertEqual(resp['status'], 404) - # Here is a more realistic test + # Works with strings too, like you get when parsing HTTP headers + # that came in through a socket from the account server cached = {'status': 404, 'bytes': '3333', 'container_count': '234', @@ -445,17 +337,17 @@ class TestFuncs(unittest.TestCase): environ={'swift.cache': FakeCache(cached)}) resp = get_account_info(req.environ, FakeApp()) self.assertEqual(resp['status'], 404) - self.assertEqual(resp['bytes'], '3333') + self.assertEqual(resp['bytes'], 3333) self.assertEqual(resp['container_count'], 234) self.assertEqual(resp['meta'], {}) - self.assertEqual(resp['total_object_count'], '10') + self.assertEqual(resp['total_object_count'], 10) def test_get_account_info_env(self): - cache_key = get_account_memcache_key("account") - env_key = 'swift.%s' % cache_key - req = Request.blank("/v1/account", - environ={env_key: {'bytes': 3867}, - 'swift.cache': FakeCache({})}) + cache_key = get_cache_key("account") + req = Request.blank( + "/v1/account", + environ={'swift.infocache': {cache_key: {'bytes': 3867}}, + 'swift.cache': FakeCache({})}) resp = get_account_info(req.environ, 'xxx') self.assertEqual(resp['bytes'], 3867) @@ -464,10 +356,11 @@ class TestFuncs(unittest.TestCase): 'length': 3333, 'type': 'application/json', 'meta': {}} - env_key = get_object_env_key("account", "cont", "obj") - req = Request.blank("/v1/account/cont/obj", - environ={env_key: cached, - 'swift.cache': FakeCache({})}) + cache_key = get_cache_key("account", "cont", "obj") + req = Request.blank( + "/v1/account/cont/obj", + environ={'swift.infocache': {cache_key: cached}, + 'swift.cache': FakeCache({})}) resp = get_object_info(req.environ, 'xxx') self.assertEqual(resp['length'], 3333) self.assertEqual(resp['type'], 'application/json') @@ -722,6 +615,8 @@ class TestFuncs(unittest.TestCase): self.assertEqual(handler.backend_headers['Range'], 'bytes=43-50') self.assertRaises(HTTPException, handler.fast_forward, 80) + self.assertRaises(exceptions.RangeAlreadyComplete, + handler.fast_forward, 8) handler = GetOrHeadHandler(None, req, None, None, None, None, {'Range': 'bytes=23-'}) @@ -732,6 +627,41 @@ class TestFuncs(unittest.TestCase): {'Range': 'bytes=-100'}) handler.fast_forward(20) self.assertEqual(handler.backend_headers['Range'], 'bytes=-80') + self.assertRaises(HTTPException, + handler.fast_forward, 100) + self.assertRaises(exceptions.RangeAlreadyComplete, + handler.fast_forward, 80) + + handler = GetOrHeadHandler(None, req, None, None, None, None, + {'Range': 'bytes=0-0'}) + self.assertRaises(exceptions.RangeAlreadyComplete, + handler.fast_forward, 1) + + def test_range_fast_forward_after_data_timeout(self): + req = Request.blank('/') + + # We get a 200 and learn that it's a 1000-byte object, but receive 0 + # bytes of data, so then we get a new node, fast_forward(0), and + # send out a new request. That new request must be for all 1000 + # bytes. + handler = GetOrHeadHandler(None, req, None, None, None, None, {}) + handler.learn_size_from_content_range(0, 999, 1000) + handler.fast_forward(0) + self.assertEqual(handler.backend_headers['Range'], 'bytes=0-999') + + # Same story as above, but a 1-byte object so we can have our byte + # indices be 0. + handler = GetOrHeadHandler(None, req, None, None, None, None, {}) + handler.learn_size_from_content_range(0, 0, 1) + handler.fast_forward(0) + self.assertEqual(handler.backend_headers['Range'], 'bytes=0-0') + + # last 100 bytes + handler = GetOrHeadHandler(None, req, None, None, None, None, + {'Range': 'bytes=-100'}) + handler.learn_size_from_content_range(900, 999, 1000) + handler.fast_forward(0) + self.assertEqual(handler.backend_headers['Range'], 'bytes=900-999') def test_transfer_headers_with_sysmeta(self): base = Controller(self.app) diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py index a95e058452..f08a90dfb1 100644 --- a/test/unit/proxy/controllers/test_container.py +++ b/test/unit/proxy/controllers/test_container.py @@ -58,7 +58,7 @@ class TestContainerController(TestRingBase): proxy_server.ContainerController): def account_info(controller, *args, **kwargs): - patch_path = 'swift.proxy.controllers.base.get_info' + patch_path = 'swift.proxy.controllers.base.get_account_info' with mock.patch(patch_path) as mock_get_info: mock_get_info.return_value = dict(self.account_info) return super(FakeAccountInfoContainerController, @@ -95,16 +95,20 @@ class TestContainerController(TestRingBase): 'Expected %s but got %s. Failed case: %s' % (expected, resp.status_int, str(responses))) - def test_container_info_in_response_env(self): + def test_container_info_got_cached(self): controller = proxy_server.ContainerController(self.app, 'a', 'c') with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200, 200, body='')): req = Request.blank('/v1/a/c', {'PATH_INFO': '/v1/a/c'}) resp = controller.HEAD(req) self.assertEqual(2, resp.status_int // 100) - self.assertTrue("swift.container/a/c" in resp.environ) - self.assertEqual(headers_to_container_info(resp.headers), - resp.environ['swift.container/a/c']) + # Make sure it's in both swift.infocache and memcache + self.assertIn("container/a/c", resp.environ['swift.infocache']) + self.assertEqual( + headers_to_container_info(resp.headers), + resp.environ['swift.infocache']['container/a/c']) + from_memcache = self.app.memcache.get('container/a/c') + self.assertTrue(from_memcache) def test_swift_owner(self): owner_headers = { diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 12f2e8889d..bd920e67b4 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -34,7 +34,8 @@ from swift.common import utils, swob, exceptions from swift.common.header_key_dict import HeaderKeyDict from swift.proxy import server as proxy_server from swift.proxy.controllers import obj -from swift.proxy.controllers.base import get_info as _real_get_info +from swift.proxy.controllers.base import \ + get_container_info as _real_get_container_info from swift.common.storage_policy import POLICIES, ECDriverError, StoragePolicy from test.unit import FakeRing, FakeMemcache, fake_http_connect, \ @@ -76,7 +77,7 @@ def set_http_connect(*args, **kwargs): class PatchedObjControllerApp(proxy_server.Application): """ This patch is just a hook over the proxy server's __call__ to ensure - that calls to get_info will return the stubbed value for + that calls to get_container_info will return the stubbed value for container_info if it's a container info call. """ @@ -85,16 +86,38 @@ class PatchedObjControllerApp(proxy_server.Application): def __call__(self, *args, **kwargs): - def _fake_get_info(app, env, account, container=None, **kwargs): - if container: - if container in self.per_container_info: - return self.per_container_info[container] - return self.container_info - else: - return _real_get_info(app, env, account, container, **kwargs) + def _fake_get_container_info(env, app, swift_source=None): + _vrs, account, container, _junk = utils.split_path( + env['PATH_INFO'], 3, 4) - mock_path = 'swift.proxy.controllers.base.get_info' - with mock.patch(mock_path, new=_fake_get_info): + # Seed the cache with our container info so that the real + # get_container_info finds it. + ic = env.setdefault('swift.infocache', {}) + cache_key = "container/%s/%s" % (account, container) + + old_value = ic.get(cache_key) + + # Copy the container info so we don't hand out a reference to a + # mutable thing that's set up only once at compile time. Nothing + # *should* mutate it, but it's better to be paranoid than wrong. + if container in self.per_container_info: + ic[cache_key] = self.per_container_info[container].copy() + else: + ic[cache_key] = self.container_info.copy() + + real_info = _real_get_container_info(env, app, swift_source) + + if old_value is None: + del ic[cache_key] + else: + ic[cache_key] = old_value + + return real_info + + with mock.patch('swift.proxy.server.get_container_info', + new=_fake_get_container_info), \ + mock.patch('swift.proxy.controllers.base.get_container_info', + new=_fake_get_container_info): return super( PatchedObjControllerApp, self).__call__(*args, **kwargs) @@ -122,6 +145,7 @@ def make_footers_callback(body): class BaseObjectControllerMixin(object): container_info = { + 'status': 200, 'write_acl': None, 'read_acl': None, 'storage_policy': None, @@ -142,8 +166,11 @@ class BaseObjectControllerMixin(object): self.app = PatchedObjControllerApp( None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), logger=self.logger) + # you can over-ride the container_info just by setting it on the app + # (see PatchedObjControllerApp for details) self.app.container_info = dict(self.container_info) + # default policy and ring references self.policy = POLICIES.default self.obj_ring = self.policy.object_ring @@ -1136,31 +1163,6 @@ class TestReplicatedObjControllerVariousReplicas(BaseObjectControllerMixin, controller_cls = obj.ReplicatedObjectController -@patch_policies(legacy_only=True) -class TestObjControllerLegacyCache(TestReplicatedObjController): - """ - This test pretends like memcache returned a stored value that should - resemble whatever "old" format. It catches KeyErrors you'd get if your - code was expecting some new format during a rolling upgrade. - """ - - # in this case policy_index is missing - container_info = { - 'read_acl': None, - 'write_acl': None, - 'sync_key': None, - 'versions': None, - } - - def test_invalid_storage_policy_cache(self): - self.app.container_info['storage_policy'] = 1 - for method in ('GET', 'HEAD', 'POST', 'PUT', 'COPY'): - req = swob.Request.blank('/v1/a/c/o', method=method) - with set_http_connect(): - resp = req.get_response(self.app) - self.assertEqual(resp.status_int, 503) - - class StubResponse(object): def __init__(self, status, body='', headers=None): @@ -1234,6 +1236,7 @@ def capture_http_requests(get_response): @patch_policies(with_ec_default=True) class TestECObjController(BaseObjectControllerMixin, unittest.TestCase): container_info = { + 'status': 200, 'read_acl': None, 'write_acl': None, 'sync_key': None, diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index f4218f617e..12f54a3c0e 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -70,8 +70,8 @@ from swift.common import utils, constraints from swift.common.utils import mkdirs, NullLogger from swift.common.wsgi import monkey_patch_mimetools, loadapp from swift.proxy.controllers import base as proxy_base -from swift.proxy.controllers.base import get_container_memcache_key, \ - get_account_memcache_key, cors_validation, _get_info_cache +from swift.proxy.controllers.base import get_cache_key, cors_validation, \ + get_account_info, get_container_info import swift.proxy.controllers import swift.proxy.controllers.obj from swift.common.header_key_dict import HeaderKeyDict @@ -285,14 +285,14 @@ class TestController(unittest.TestCase): self.controller.account_info(self.account) self.assertEqual(count, 123) with save_globals(): - cache_key = get_account_memcache_key(self.account) + cache_key = get_cache_key(self.account) account_info = {'status': 200, 'container_count': 1234} self.memcache.set(cache_key, account_info) partition, nodes, count = \ self.controller.account_info(self.account) self.assertEqual(count, 1234) with save_globals(): - cache_key = get_account_memcache_key(self.account) + cache_key = get_cache_key(self.account) account_info = {'status': 200, 'container_count': '1234'} self.memcache.set(cache_key, account_info) partition, nodes, count = \ @@ -320,8 +320,9 @@ class TestController(unittest.TestCase): # Test the internal representation in memcache # 'container_count' changed from int to str - cache_key = get_account_memcache_key(self.account) + cache_key = get_cache_key(self.account) container_info = {'status': 200, + 'account_really_exists': True, 'container_count': '12345', 'total_object_count': None, 'bytes': None, @@ -347,7 +348,7 @@ class TestController(unittest.TestCase): # Test the internal representation in memcache # 'container_count' changed from 0 to None - cache_key = get_account_memcache_key(self.account) + cache_key = get_cache_key(self.account) account_info = {'status': 404, 'container_count': None, # internally keep None 'total_object_count': None, @@ -424,8 +425,7 @@ class TestController(unittest.TestCase): self.account, self.container, self.request) self.check_container_info_return(ret) - cache_key = get_container_memcache_key(self.account, - self.container) + cache_key = get_cache_key(self.account, self.container) cache_value = self.memcache.get(cache_key) self.assertTrue(isinstance(cache_value, dict)) self.assertEqual(200, cache_value.get('status')) @@ -447,8 +447,7 @@ class TestController(unittest.TestCase): self.account, self.container, self.request) self.check_container_info_return(ret, True) - cache_key = get_container_memcache_key(self.account, - self.container) + cache_key = get_cache_key(self.account, self.container) cache_value = self.memcache.get(cache_key) self.assertTrue(isinstance(cache_value, dict)) self.assertEqual(404, cache_value.get('status')) @@ -463,8 +462,7 @@ class TestController(unittest.TestCase): self.account, self.container, self.request) self.check_container_info_return(ret, True) - cache_key = get_container_memcache_key(self.account, - self.container) + cache_key = get_cache_key(self.account, self.container) cache_value = self.memcache.get(cache_key) self.assertTrue(isinstance(cache_value, dict)) self.assertEqual(404, cache_value.get('status')) @@ -490,7 +488,32 @@ class TestController(unittest.TestCase): test(404, 507, 503) test(503, 503, 503) - def test_get_info_cache_returns_values_as_strings(self): + def test_get_account_info_returns_values_as_strings(self): + app = mock.MagicMock() + app.memcache = mock.MagicMock() + app.memcache.get = mock.MagicMock() + app.memcache.get.return_value = { + u'foo': u'\u2603', + u'meta': {u'bar': u'\u2603'}, + u'sysmeta': {u'baz': u'\u2603'}} + env = {'PATH_INFO': '/v1/a'} + ai = get_account_info(env, app) + + # Test info is returned as strings + self.assertEqual(ai.get('foo'), '\xe2\x98\x83') + self.assertTrue(isinstance(ai.get('foo'), str)) + + # Test info['meta'] is returned as strings + m = ai.get('meta', {}) + self.assertEqual(m.get('bar'), '\xe2\x98\x83') + self.assertTrue(isinstance(m.get('bar'), str)) + + # Test info['sysmeta'] is returned as strings + m = ai.get('sysmeta', {}) + self.assertEqual(m.get('baz'), '\xe2\x98\x83') + self.assertTrue(isinstance(m.get('baz'), str)) + + def test_get_container_info_returns_values_as_strings(self): app = mock.MagicMock() app.memcache = mock.MagicMock() app.memcache.get = mock.MagicMock() @@ -499,25 +522,25 @@ class TestController(unittest.TestCase): u'meta': {u'bar': u'\u2603'}, u'sysmeta': {u'baz': u'\u2603'}, u'cors': {u'expose_headers': u'\u2603'}} - env = {} - r = _get_info_cache(app, env, 'account', 'container') + env = {'PATH_INFO': '/v1/a/c'} + ci = get_container_info(env, app) # Test info is returned as strings - self.assertEqual(r.get('foo'), '\xe2\x98\x83') - self.assertTrue(isinstance(r.get('foo'), str)) + self.assertEqual(ci.get('foo'), '\xe2\x98\x83') + self.assertTrue(isinstance(ci.get('foo'), str)) # Test info['meta'] is returned as strings - m = r.get('meta', {}) + m = ci.get('meta', {}) self.assertEqual(m.get('bar'), '\xe2\x98\x83') self.assertTrue(isinstance(m.get('bar'), str)) # Test info['sysmeta'] is returned as strings - m = r.get('sysmeta', {}) + m = ci.get('sysmeta', {}) self.assertEqual(m.get('baz'), '\xe2\x98\x83') self.assertTrue(isinstance(m.get('baz'), str)) # Test info['cors'] is returned as strings - m = r.get('cors', {}) + m = ci.get('cors', {}) self.assertEqual(m.get('expose_headers'), '\xe2\x98\x83') self.assertTrue(isinstance(m.get('expose_headers'), str)) @@ -6141,22 +6164,23 @@ class TestContainerController(unittest.TestCase): res = controller.HEAD(req) self.assertEqual(res.status[:len(str(expected))], str(expected)) + infocache = res.environ.get('swift.infocache', {}) if expected < 400: - self.assertTrue('x-works' in res.headers) + self.assertIn('x-works', res.headers) self.assertEqual(res.headers['x-works'], 'yes') if c_expected: - self.assertTrue('swift.container/a/c' in res.environ) + self.assertIn('container/a/c', infocache) self.assertEqual( - res.environ['swift.container/a/c']['status'], + infocache['container/a/c']['status'], c_expected) else: - self.assertTrue('swift.container/a/c' not in res.environ) + self.assertNotIn('container/a/c', infocache) if a_expected: - self.assertTrue('swift.account/a' in res.environ) - self.assertEqual(res.environ['swift.account/a']['status'], + self.assertIn('account/a', infocache) + self.assertEqual(infocache['account/a']['status'], a_expected) else: - self.assertTrue('swift.account/a' not in res.environ) + self.assertNotIn('account/a', res.environ) set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} @@ -6165,25 +6189,26 @@ class TestContainerController(unittest.TestCase): res = controller.GET(req) self.assertEqual(res.status[:len(str(expected))], str(expected)) + infocache = res.environ.get('swift.infocache', {}) if expected < 400: self.assertTrue('x-works' in res.headers) self.assertEqual(res.headers['x-works'], 'yes') if c_expected: - self.assertTrue('swift.container/a/c' in res.environ) + self.assertIn('container/a/c', infocache) self.assertEqual( - res.environ['swift.container/a/c']['status'], + infocache['container/a/c']['status'], c_expected) else: - self.assertTrue('swift.container/a/c' not in res.environ) + self.assertNotIn('container/a/c', infocache) if a_expected: - self.assertTrue('swift.account/a' in res.environ) - self.assertEqual(res.environ['swift.account/a']['status'], + self.assertIn('account/a', infocache) + self.assertEqual(infocache['account/a']['status'], a_expected) else: - self.assertTrue('swift.account/a' not in res.environ) + self.assertNotIn('account/a', infocache) # In all the following tests cache 200 for account - # return and ache vary for container - # return 200 and cache 200 for and container + # return and cache vary for container + # return 200 and cache 200 for account and container test_status_map((200, 200, 404, 404), 200, 200, 200) test_status_map((200, 200, 500, 404), 200, 200, 200) # return 304 don't cache container @@ -6195,12 +6220,13 @@ class TestContainerController(unittest.TestCase): test_status_map((200, 500, 500, 500), 503, None, 200) self.assertFalse(self.app.account_autocreate) - # In all the following tests cache 404 for account # return 404 (as account is not found) and don't cache container test_status_map((404, 404, 404), 404, None, 404) - # This should make no difference + + # cache a 204 for the account because it's sort of like it + # exists self.app.account_autocreate = True - test_status_map((404, 404, 404), 404, None, 404) + test_status_map((404, 404, 404), 404, None, 204) def test_PUT_policy_headers(self): backend_requests = [] @@ -6786,14 +6812,13 @@ class TestContainerController(unittest.TestCase): def test_GET_no_content(self): with save_globals(): set_http_connect(200, 204, 204, 204) - controller = proxy_server.ContainerController(self.app, 'account', - 'container') + controller = proxy_server.ContainerController(self.app, 'a', 'c') req = Request.blank('/v1/a/c') self.app.update_request(req) res = controller.GET(req) self.assertEqual(res.status_int, 204) - self.assertEqual( - res.environ['swift.container/a/c']['status'], 204) + ic = res.environ['swift.infocache'] + self.assertEqual(ic['container/a/c']['status'], 204) self.assertEqual(res.content_length, 0) self.assertTrue('transfer-encoding' not in res.headers) @@ -6805,13 +6830,14 @@ class TestContainerController(unittest.TestCase): return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 201, 201, 201) - controller = proxy_server.ContainerController(self.app, 'account', - 'container') + controller = proxy_server.ContainerController(self.app, 'a', 'c') req = Request.blank('/v1/a/c') req.environ['swift.authorize'] = authorize self.app.update_request(req) res = controller.GET(req) - self.assertEqual(res.environ['swift.container/a/c']['status'], 201) + self.assertEqual( + res.environ['swift.infocache']['container/a/c']['status'], + 201) self.assertTrue(called[0]) def test_HEAD_calls_authorize(self): @@ -6822,8 +6848,7 @@ class TestContainerController(unittest.TestCase): return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 201, 201, 201) - controller = proxy_server.ContainerController(self.app, 'account', - 'container') + controller = proxy_server.ContainerController(self.app, 'a', 'c') req = Request.blank('/v1/a/c', {'REQUEST_METHOD': 'HEAD'}) req.environ['swift.authorize'] = authorize self.app.update_request(req) @@ -7279,16 +7304,18 @@ class TestAccountController(unittest.TestCase): self.app.update_request(req) res = method(req) self.assertEqual(res.status_int, expected) + infocache = res.environ.get('swift.infocache', {}) if env_expected: - self.assertEqual(res.environ['swift.account/a']['status'], + self.assertEqual(infocache['account/a']['status'], env_expected) set_http_connect(*statuses) req = Request.blank('/v1/a/', {}) self.app.update_request(req) res = method(req) + infocache = res.environ.get('swift.infocache', {}) self.assertEqual(res.status_int, expected) if env_expected: - self.assertEqual(res.environ['swift.account/a']['status'], + self.assertEqual(infocache['account/a']['status'], env_expected) def test_OPTIONS(self): @@ -7333,7 +7360,7 @@ class TestAccountController(unittest.TestCase): def test_GET(self): with save_globals(): - controller = proxy_server.AccountController(self.app, 'account') + controller = proxy_server.AccountController(self.app, 'a') # GET returns after the first successful call to an Account Server self.assert_status_map(controller.GET, (200,), 200, 200) self.assert_status_map(controller.GET, (503, 200), 200, 200) @@ -7355,7 +7382,7 @@ class TestAccountController(unittest.TestCase): def test_GET_autocreate(self): with save_globals(): - controller = proxy_server.AccountController(self.app, 'account') + controller = proxy_server.AccountController(self.app, 'a') self.app.memcache = FakeMemcacheReturnsNone() self.assertFalse(self.app.account_autocreate) # Repeat the test for autocreate = False and 404 by all @@ -7380,7 +7407,7 @@ class TestAccountController(unittest.TestCase): def test_HEAD(self): # Same behaviour as GET with save_globals(): - controller = proxy_server.AccountController(self.app, 'account') + controller = proxy_server.AccountController(self.app, 'a') self.assert_status_map(controller.HEAD, (200,), 200, 200) self.assert_status_map(controller.HEAD, (503, 200), 200, 200) self.assert_status_map(controller.HEAD, (503, 503, 200), 200, 200) @@ -7398,7 +7425,7 @@ class TestAccountController(unittest.TestCase): def test_HEAD_autocreate(self): # Same behaviour as GET with save_globals(): - controller = proxy_server.AccountController(self.app, 'account') + controller = proxy_server.AccountController(self.app, 'a') self.app.memcache = FakeMemcacheReturnsNone() self.assertFalse(self.app.account_autocreate) self.assert_status_map(controller.HEAD, @@ -7414,7 +7441,7 @@ class TestAccountController(unittest.TestCase): def test_POST_autocreate(self): with save_globals(): - controller = proxy_server.AccountController(self.app, 'account') + controller = proxy_server.AccountController(self.app, 'a') self.app.memcache = FakeMemcacheReturnsNone() # first test with autocreate being False self.assertFalse(self.app.account_autocreate) @@ -7436,7 +7463,7 @@ class TestAccountController(unittest.TestCase): def test_POST_autocreate_with_sysmeta(self): with save_globals(): - controller = proxy_server.AccountController(self.app, 'account') + controller = proxy_server.AccountController(self.app, 'a') self.app.memcache = FakeMemcacheReturnsNone() # first test with autocreate being False self.assertFalse(self.app.account_autocreate) diff --git a/test/unit/proxy/test_sysmeta.py b/test/unit/proxy/test_sysmeta.py index 2060b1d5ab..7eba2c8760 100644 --- a/test/unit/proxy/test_sysmeta.py +++ b/test/unit/proxy/test_sysmeta.py @@ -19,6 +19,8 @@ import unittest import os from tempfile import mkdtemp import shutil + +from swift.common.middleware.copy import ServerSideCopyMiddleware from swift.common.storage_policy import StoragePolicy from swift.common.swob import Request from swift.common.utils import mkdirs, split_path @@ -134,6 +136,7 @@ class TestObjectSysmeta(unittest.TestCase): logger=debug_logger('proxy-ut'), account_ring=FakeRing(replicas=1), container_ring=FakeRing(replicas=1)) + self.copy_app = ServerSideCopyMiddleware(self.app, {}) monkey_patch_mimetools() self.tmpdir = mkdtemp() self.testdir = os.path.join(self.tmpdir, @@ -253,7 +256,7 @@ class TestObjectSysmeta(unittest.TestCase): self._assertInHeaders(resp, self.new_meta_headers) self._assertNotInHeaders(resp, self.original_meta_headers_2) - def test_sysmeta_not_updated_by_POST(self): + def _test_sysmeta_not_updated_by_POST(self, app): # check sysmeta is not changed by a POST but user meta is replaced path = '/v1/a/c/o' @@ -261,7 +264,7 @@ class TestObjectSysmeta(unittest.TestCase): hdrs = dict(self.original_sysmeta_headers_1) hdrs.update(self.original_meta_headers_1) req = Request.blank(path, environ=env, headers=hdrs, body='x') - resp = req.get_response(self.app) + resp = req.get_response(app) self._assertStatus(resp, 201) env = {'REQUEST_METHOD': 'POST'} @@ -271,11 +274,11 @@ class TestObjectSysmeta(unittest.TestCase): hdrs.update(self.new_meta_headers) hdrs.update(self.bad_headers) req = Request.blank(path, environ=env, headers=hdrs) - resp = req.get_response(self.app) + resp = req.get_response(app) self._assertStatus(resp, 202) req = Request.blank(path, environ={}) - resp = req.get_response(self.app) + resp = req.get_response(app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.original_sysmeta_headers_1) self._assertNotInHeaders(resp, self.new_sysmeta_headers) @@ -288,17 +291,121 @@ class TestObjectSysmeta(unittest.TestCase): hdrs.update(self.new_sysmeta_headers) hdrs.update(self.bad_headers) req = Request.blank(path, environ=env, headers=hdrs, body='x') - resp = req.get_response(self.app) + resp = req.get_response(app) self._assertStatus(resp, 201) req = Request.blank(path, environ={}) - resp = req.get_response(self.app) + resp = req.get_response(app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.changed_sysmeta_headers) self._assertInHeaders(resp, self.new_sysmeta_headers) self._assertNotInHeaders(resp, self.original_sysmeta_headers_2) - def test_transient_sysmeta_replaced_by_PUT_or_POST(self): + def test_sysmeta_not_updated_by_POST(self): + # test fast-post by issuing requests to the proxy app + self._test_sysmeta_not_updated_by_POST(self.app) + + def test_sysmeta_not_updated_by_POST_as_copy(self): + # test post-as-copy by issuing requests to the copy middleware app + self.copy_app.object_post_as_copy = True + self._test_sysmeta_not_updated_by_POST(self.copy_app) + + def test_sysmeta_updated_by_COPY(self): + # check sysmeta is updated by a COPY in same way as user meta by + # issuing requests to the copy middleware app + path = '/v1/a/c/o' + dest = '/c/o2' + env = {'REQUEST_METHOD': 'PUT'} + hdrs = dict(self.original_sysmeta_headers_1) + hdrs.update(self.original_sysmeta_headers_2) + hdrs.update(self.original_meta_headers_1) + hdrs.update(self.original_meta_headers_2) + hdrs.update(self.original_transient_sysmeta_headers_1) + hdrs.update(self.original_transient_sysmeta_headers_2) + req = Request.blank(path, environ=env, headers=hdrs, body='x') + resp = req.get_response(self.copy_app) + self._assertStatus(resp, 201) + + env = {'REQUEST_METHOD': 'COPY'} + hdrs = dict(self.changed_sysmeta_headers) + hdrs.update(self.new_sysmeta_headers) + hdrs.update(self.changed_meta_headers) + hdrs.update(self.new_meta_headers) + hdrs.update(self.changed_transient_sysmeta_headers) + hdrs.update(self.new_transient_sysmeta_headers) + hdrs.update(self.bad_headers) + hdrs.update({'Destination': dest}) + req = Request.blank(path, environ=env, headers=hdrs) + resp = req.get_response(self.copy_app) + self._assertStatus(resp, 201) + self._assertInHeaders(resp, self.changed_sysmeta_headers) + self._assertInHeaders(resp, self.new_sysmeta_headers) + self._assertInHeaders(resp, self.original_sysmeta_headers_2) + self._assertInHeaders(resp, self.changed_meta_headers) + self._assertInHeaders(resp, self.new_meta_headers) + self._assertInHeaders(resp, self.original_meta_headers_2) + self._assertInHeaders(resp, self.changed_transient_sysmeta_headers) + self._assertInHeaders(resp, self.new_transient_sysmeta_headers) + self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2) + self._assertNotInHeaders(resp, self.bad_headers) + + req = Request.blank('/v1/a/c/o2', environ={}) + resp = req.get_response(self.copy_app) + self._assertStatus(resp, 200) + self._assertInHeaders(resp, self.changed_sysmeta_headers) + self._assertInHeaders(resp, self.new_sysmeta_headers) + self._assertInHeaders(resp, self.original_sysmeta_headers_2) + self._assertInHeaders(resp, self.changed_meta_headers) + self._assertInHeaders(resp, self.new_meta_headers) + self._assertInHeaders(resp, self.original_meta_headers_2) + self._assertInHeaders(resp, self.changed_transient_sysmeta_headers) + self._assertInHeaders(resp, self.new_transient_sysmeta_headers) + self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2) + self._assertNotInHeaders(resp, self.bad_headers) + + def test_sysmeta_updated_by_COPY_from(self): + # check sysmeta is updated by a PUT with x-copy-from in same way as + # user meta by issuing requests to the copy middleware app + path = '/v1/a/c/o' + env = {'REQUEST_METHOD': 'PUT'} + hdrs = dict(self.original_sysmeta_headers_1) + hdrs.update(self.original_sysmeta_headers_2) + hdrs.update(self.original_meta_headers_1) + hdrs.update(self.original_meta_headers_2) + req = Request.blank(path, environ=env, headers=hdrs, body='x') + resp = req.get_response(self.copy_app) + self._assertStatus(resp, 201) + + env = {'REQUEST_METHOD': 'PUT'} + hdrs = dict(self.changed_sysmeta_headers) + hdrs.update(self.new_sysmeta_headers) + hdrs.update(self.changed_meta_headers) + hdrs.update(self.new_meta_headers) + hdrs.update(self.bad_headers) + hdrs.update({'X-Copy-From': '/c/o'}) + req = Request.blank('/v1/a/c/o2', environ=env, headers=hdrs, body='') + resp = req.get_response(self.copy_app) + self._assertStatus(resp, 201) + self._assertInHeaders(resp, self.changed_sysmeta_headers) + self._assertInHeaders(resp, self.new_sysmeta_headers) + self._assertInHeaders(resp, self.original_sysmeta_headers_2) + self._assertInHeaders(resp, self.changed_meta_headers) + self._assertInHeaders(resp, self.new_meta_headers) + self._assertInHeaders(resp, self.original_meta_headers_2) + self._assertNotInHeaders(resp, self.bad_headers) + + req = Request.blank('/v1/a/c/o2', environ={}) + resp = req.get_response(self.copy_app) + self._assertStatus(resp, 200) + self._assertInHeaders(resp, self.changed_sysmeta_headers) + self._assertInHeaders(resp, self.new_sysmeta_headers) + self._assertInHeaders(resp, self.original_sysmeta_headers_2) + self._assertInHeaders(resp, self.changed_meta_headers) + self._assertInHeaders(resp, self.new_meta_headers) + self._assertInHeaders(resp, self.original_meta_headers_2) + self._assertNotInHeaders(resp, self.bad_headers) + + def _test_transient_sysmeta_replaced_by_PUT_or_POST(self, app): # check transient_sysmeta is replaced en-masse by a POST path = '/v1/a/c/o' @@ -307,17 +414,17 @@ class TestObjectSysmeta(unittest.TestCase): hdrs.update(self.original_transient_sysmeta_headers_2) hdrs.update(self.original_meta_headers_1) req = Request.blank(path, environ=env, headers=hdrs, body='x') - resp = req.get_response(self.app) + resp = req.get_response(app) self._assertStatus(resp, 201) req = Request.blank(path, environ={}) - resp = req.get_response(self.app) + resp = req.get_response(app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.original_transient_sysmeta_headers_1) self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2) self._assertInHeaders(resp, self.original_meta_headers_1) - info = get_object_info(req.environ, self.app) + info = get_object_info(req.environ, app) self.assertEqual(2, len(info.get('transient_sysmeta', ()))) self.assertEqual({'testa': 'A', 'testb': 'B'}, info['transient_sysmeta']) @@ -327,11 +434,11 @@ class TestObjectSysmeta(unittest.TestCase): hdrs = dict(self.changed_transient_sysmeta_headers) hdrs.update(self.new_transient_sysmeta_headers) req = Request.blank(path, environ=env, headers=hdrs) - resp = req.get_response(self.app) + resp = req.get_response(app) self._assertStatus(resp, 202) req = Request.blank(path, environ={}) - resp = req.get_response(self.app) + resp = req.get_response(app) self._assertStatus(resp, 200) self._assertNotInHeaders(resp, self.original_meta_headers_2) self._assertInHeaders(resp, self.changed_transient_sysmeta_headers) @@ -339,7 +446,7 @@ class TestObjectSysmeta(unittest.TestCase): self._assertNotInHeaders(resp, self.original_transient_sysmeta_headers_2) - info = get_object_info(req.environ, self.app) + info = get_object_info(req.environ, app) self.assertEqual(2, len(info.get('transient_sysmeta', ()))) self.assertEqual({'testa': 'changed_A', 'testc': 'C'}, info['transient_sysmeta']) @@ -349,11 +456,11 @@ class TestObjectSysmeta(unittest.TestCase): hdrs = dict(self.original_transient_sysmeta_headers_2) hdrs.update(self.original_meta_headers_2) req = Request.blank(path, environ=env, headers=hdrs, body='x') - resp = req.get_response(self.app) + resp = req.get_response(app) self._assertStatus(resp, 201) req = Request.blank(path, environ={}) - resp = req.get_response(self.app) + resp = req.get_response(app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.original_meta_headers_2) self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2) @@ -362,6 +469,14 @@ class TestObjectSysmeta(unittest.TestCase): self._assertNotInHeaders(resp, self.changed_transient_sysmeta_headers) self._assertNotInHeaders(resp, self.new_transient_sysmeta_headers) - info = get_object_info(req.environ, self.app) + info = get_object_info(req.environ, app) self.assertEqual(1, len(info.get('transient_sysmeta', ()))) self.assertEqual({'testb': 'B'}, info['transient_sysmeta']) + + def test_transient_sysmeta_replaced_by_PUT_or_POST(self): + self._test_transient_sysmeta_replaced_by_PUT_or_POST(self.app) + + def test_transient_sysmeta_replaced_by_PUT_or_POST_as_copy(self): + # test post-as-copy by issuing requests to the copy middleware app + self.copy_app.object_post_as_copy = True + self._test_transient_sysmeta_replaced_by_PUT_or_POST(self.copy_app) diff --git a/tox.ini b/tox.ini index f016316c49..1e79f67b88 100644 --- a/tox.ini +++ b/tox.ini @@ -76,7 +76,14 @@ commands = bandit -c bandit.yaml -r swift bin -n 5 -p gate # H404: multi line docstring should start without a leading new line # H405: multi line docstring summary not separated with an empty line # H501: Do not use self.__dict__ for string formatting -# H703: Multiple positional placeholders -ignore = F812,H101,H202,H233,H301,H306,H401,H403,H404,H405,H501,H703 +ignore = F812,H101,H202,H233,H301,H306,H401,H403,H404,H405,H501 exclude = .venv,.tox,dist,*egg show-source = True + +[testenv:bindep] +# Do not install any requirements. We want this to be fast and work even if +# system dependencies are missing, since it's used to tell you what system +# dependencies are missing! This also means that bindep must be installed +# separately, outside of the requirements files. +deps = bindep +commands = bindep test