openstack
/
swift
Code Issues Proposed changes

get_info - removes duplicate code (Take 3) 

Consolidate the different ways in which info of account/container
is gathered, cached, used, updated, etc.

This refactoring increases code reuse and is a basis for later
addition of account ACLs.
Changing the get_info users is left for future.
This staged approach ensures the behaviour is unchanged.

Change-Id: I67b58030d3f9e3bc86bcd7ece0f1dc693c4e08c3
Fixes: Bug #1162199
This commit is contained in:
David Hadas 2013-03-30 15:55:29 +03:00
parent 03c0c5d658
commit 58f4e0f2e0
10 changed files with 809 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
swift/proxy/controllers/account.py
View File

@ -31,7 +31,7 @@ from swift.account.utils import account_listing_response, \
from swift.common.utils import public
from swift.common.constraints import check_metadata, MAX_ACCOUNT_NAME_LENGTH
from swift.common.http import HTTP_NOT_FOUND
from swift.proxy.controllers.base import Controller, get_account_memcache_key
from swift.proxy.controllers.base import Controller, clear_info_cache
from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed
@ -84,9 +84,7 @@ class AccountController(Controller):
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = self.generate_request_headers(req, transfer=True)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
clear_info_cache(self.app, req.environ, self.account_name)
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'PUT',
req.path_info, [headers] * len(accounts))
@ -106,14 +104,12 @@ class AccountController(Controller):
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = self.generate_request_headers(req, transfer=True)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
clear_info_cache(self.app, req.environ, self.account_name)
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'POST',
req.path_info, [headers] * len(accounts))
if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate:
self.autocreate_account(self.account_name)
self.autocreate_account(req.environ, self.account_name)
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'POST',
req.path_info, [headers] * len(accounts))
@ -134,9 +130,7 @@ class AccountController(Controller):
account_partition, accounts = \
self.app.account_ring.get_nodes(self.account_name)
headers = self.generate_request_headers(req)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(self.account_name))
clear_info_cache(self.app, req.environ, self.account_name)
resp = self.make_requests(
req, self.app.account_ring, account_partition, 'DELETE',
req.path_info, [headers] * len(accounts))

499
swift/proxy/controllers/base.py
View File

@ -28,15 +28,15 @@ import os
import time
import functools
import inspect
from urllib import quote
from eventlet import spawn_n, GreenPile
from eventlet.queue import Queue, Empty, Full
from eventlet.timeout import Timeout
from swift.common.wsgi import make_pre_authed_request
from swift.common.wsgi import make_pre_authed_env
from swift.common.utils import normalize_timestamp, config_true_value, \
public, split_path, cache_from_env, list_from_csv, \
GreenthreadSafeIterator
public, split_path, list_from_csv, GreenthreadSafeIterator
from swift.common.bufferedhttp import http_connect
from swift.common.exceptions import ChunkReadTimeout, ConnectionTimeout
from swift.common.http import is_informational, is_success, is_redirection, \
@ -91,11 +91,15 @@ def delay_denial(func):
def get_account_memcache_key(account):
return 'account/%s' % account
cache_key, env_key = _get_cache_key(account, None)
return cache_key
def get_container_memcache_key(account, container):
return 'container/%s/%s' % (account, container)
if not container:
raise ValueError("container not provided")
cache_key, env_key = _get_cache_key(account, container)
return cache_key
def headers_to_account_info(headers, status_int=HTTP_OK):
@ -105,6 +109,10 @@ def headers_to_account_info(headers, status_int=HTTP_OK):
headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
return {
'status': status_int,
# 'container_count' anomaly:
# Previous code sometimes expects an int sometimes a string
# Current code aligns to str and None, yet translates to int in
# deprecated functions as needed
'container_count': headers.get('x-account-container-count'),
'total_object_count': headers.get('x-account-object-count'),
'bytes': headers.get('x-account-bytes-used'),
@ -162,7 +170,7 @@ def cors_validation(func):
# Yes, this is a CORS request so test if the origin is allowed
container_info = \
controller.container_info(controller.account_name,
controller.container_name)
controller.container_name, req)
cors_info = container_info.get('cors', {})
# Call through to the decorated method
@ -207,28 +215,14 @@ def get_container_info(env, app, swift_source=None):
Get the info structure for a container, based on env and app.
This is useful to middlewares.
Note: This call bypasses auth. Success does not imply that the
request has authorization to the container_info.
request has authorization to the account.
"""
cache = cache_from_env(env)
if not cache:
return None
(version, account, container, _) = \
split_path(env['PATH_INFO'], 3, 4, True)
cache_key = get_container_memcache_key(account, container)
# Use a unique environment cache key per container. If you copy this env
# to make a new request, it won't accidentally reuse the old container info
env_key = 'swift.%s' % cache_key
if env_key not in env:
container_info = cache.get(cache_key)
if not container_info:
resp = make_pre_authed_request(
env, 'HEAD', '/%s/%s/%s' % (version, account, container),
swift_source=swift_source,
).get_response(app)
container_info = headers_to_container_info(
resp.headers, resp.status_int)
env[env_key] = container_info
return env[env_key]
info = get_info(app, env, account, container, ret_not_found=True)
if not info:
info = headers_to_container_info({}, 0)
return info
def get_account_info(env, app, swift_source=None):
@ -236,28 +230,177 @@ def get_account_info(env, app, swift_source=None):
Get the info structure for an account, based on env and app.
This is useful to middlewares.
Note: This call bypasses auth. Success does not imply that the
request has authorization to the account_info.
request has authorization to the container.
"""
cache = cache_from_env(env)
if not cache:
return None
(version, account, _junk, _junk) = \
split_path(env['PATH_INFO'], 2, 4, True)
cache_key = get_account_memcache_key(account)
# Use a unique environment cache key per account. If you copy this env
# to make a new request, it won't accidentally reuse the old account info
info = get_info(app, env, account, ret_not_found=True)
if not info:
info = headers_to_account_info({}, 0)
if info.get('container_count') is None:
info['container_count'] = 0
else:
info['container_count'] = int(info['container_count'])
return info
def _get_cache_key(account, container):
"""
Get the keys for both memcache (cache_key) and env (env_key)
where info about accounts and containers is cached
:param account: The name of the account
:param container: The name of the container (or None if account)
:returns a tuple of (cache_key, env_key)
"""
if container:
cache_key = 'container/%s/%s' % (account, container)
else:
cache_key = 'account/%s' % account
# Use a unique environment cache key per account and one container.
# This allows caching both account and container and ensures that when we
# copy this env to form a new request, it won't accidentally reuse the
# old container or account info
env_key = 'swift.%s' % cache_key
if env_key not in env:
account_info = cache.get(cache_key)
if not account_info:
resp = make_pre_authed_request(
env, 'HEAD', '/%s/%s' % (version, account),
swift_source=swift_source,
).get_response(app)
account_info = headers_to_account_info(
resp.headers, resp.status_int)
env[env_key] = account_info
return env[env_key]
return cache_key, env_key
def _set_info_cache(app, env, account, container, resp):
"""
Cache info in both memcache and env.
Caching is used to avoid unnecessary calls to account & container servers.
This is a private function that is being called by GETorHEAD_base and
by clear_info_cache.
Any attempt to GET or HEAD from the container/account server should use
the GETorHEAD_base interface which would than set the cache.
:param app: the application object
:param account: the unquoted account name
:param container: the unquoted containr name or None
:param resp: the response received or None if info cache should be cleared
"""
if container:
cache_time = app.recheck_container_existence
else:
cache_time = app.recheck_account_existence
cache_key, env_key = _get_cache_key(account, container)
if resp:
if resp.status_int == HTTP_NOT_FOUND:
cache_time *= 0.1
elif not is_success(resp.status_int):
cache_time = None
else:
cache_time = None
# Next actually set both memcache and the env chache
memcache = getattr(app, 'memcache', None) or env.get('swift.cache')
if not cache_time:
env.pop(env_key, None)
if memcache:
memcache.delete(cache_key)
return
if container:
info = headers_to_container_info(resp.headers, resp.status_int)
else:
info = headers_to_account_info(resp.headers, resp.status_int)
if memcache:
memcache.set(cache_key, info, cache_time)
env[env_key] = info
def clear_info_cache(app, env, account, container=None):
"""
Clear the cached info in both memcache and env
:param app: the application object
:param account: the account name
:param container: the containr name or None if setting info for containers
"""
_set_info_cache(app, env, account, container, None)
def _get_info_cache(app, env, account, container=None):
"""
Get the cached info from env or memcache (if used) in that order
Used for both account and container info
A private function used by get_info
:param app: the application object
:param env: the environment used by the current request
:returns the cached info or None if not cached
"""
cache_key, env_key = _get_cache_key(account, container)
if env_key in env:
return env[env_key]
memcache = getattr(app, 'memcache', None) or env.get('swift.cache')
if memcache:
info = memcache.get(cache_key)
if info:
env[env_key] = info
return info
return None
def _prepare_pre_auth_info_request(env, path):
"""
Prepares a pre authed request to obtain info using a HEAD.
:param env: the environment used by the current request
:param path: The unquoted request path
:returns: the pre authed request
"""
# Set the env for the pre_authed call without a query string
newenv = make_pre_authed_env(env, 'HEAD', path, agent='Swift',
query_string='', swift_source='GET_INFO')
# Note that Request.blank expects quoted path
return Request.blank(quote(path), environ=newenv)
def get_info(app, env, account, container=None, ret_not_found=False):
"""
Get the info about accounts or containers
Note: This call bypasses auth. Success does not imply that the
request has authorization to the info.
:param app: the application object
:param env: the environment used by the current request
:param account: The unquoted name of the account
:param container: The unquoted name of the container (or None if account)
:returns: the cached info or None if cannot be retrieved
"""
info = _get_info_cache(app, env, account, container)
if info:
if ret_not_found or is_success(info['status']):
return info
return None
# Not in cached, let's try the account servers
path = '/v1/%s' % account
if container:
# Stop and check if we have an account?
if not get_info(app, env, account):
return None
path += '/' + container
req = _prepare_pre_auth_info_request(env, path)
# Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in
# the environment under environ[env_key] and in memcache. We will
# pick the one from environ[env_key] and use it to set the caller env
resp = req.get_response(app)
cache_key, env_key = _get_cache_key(account, container)
try:
info = resp.environ[env_key]
env[env_key] = info
if ret_not_found or is_success(info['status']):
return info
except (KeyError, AttributeError):
pass
return None
class Controller(object):
@ -268,6 +411,11 @@ class Controller(object):
pass_through_headers = []
def __init__(self, app):
"""
Creates a controller attached to an application instance
:param app: the application instance
"""
self.account_name = None
self.app = app
self.trans_id = '-'
@ -278,9 +426,21 @@ class Controller(object):
self.allowed_methods.add(name)
def _x_remove_headers(self):
"""
Returns a list of headers that must not be sent to the backend
:returns: a list of header
"""
return []
def transfer_headers(self, src_headers, dst_headers):
"""
Transfer legal headers from an original client request to dictionary
that will be used as headers by the backend request
:param src_headers: A dictionary of the original client request headers
:param dst_headers: A dictionary of the backend request headers
"""
st = self.server_type.lower()
x_remove = 'x-remove-%s-meta-' % st
@ -297,6 +457,14 @@ class Controller(object):
def generate_request_headers(self, orig_req=None, additional=None,
transfer=False):
"""
Create a list of headers to be used in backend requets
:param orig_req: the original request sent by the client to the proxy
:param additional: additional headers to send to the backend
:param transfer: If True, transfer headers from original client request
:returns: a dictionary of headers
"""
# Use the additional headers first so they don't overwrite the headers
# we require.
headers = HeaderKeyDict(additional) if additional else HeaderKeyDict()
@ -385,101 +553,31 @@ class Controller(object):
:param account: name of the account to get the info for
:param req: caller's HTTP request context object (optional)
:param autocreate: whether or not to automatically create the given
account or not (optional, default: False)
:returns: tuple of (account partition, account nodes, container_count)
or (None, None, None) if it does not exist
"""
partition, nodes = self.app.account_ring.get_nodes(account)
account_info = {'status': 0,
'container_count': 0,
'total_object_count': None,
'bytes': None,
'meta': {}}
# 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
if self.app.memcache:
cache_key = get_account_memcache_key(account)
cache_value = self.app.memcache.get(cache_key)
if not isinstance(cache_value, dict):
result_code = cache_value
container_count = 0
else:
result_code = cache_value['status']
try:
container_count = int(cache_value['container_count'])
except ValueError:
container_count = 0
if result_code == HTTP_OK:
return partition, nodes, container_count
elif result_code == HTTP_NOT_FOUND:
return None, None, None
result_code = 0
path = '/%s' % account
headers = self.generate_request_headers(req)
for node in self.iter_nodes(self.app.account_ring, partition):
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], partition, 'HEAD',
path, headers)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
body = resp.read()
if is_success(resp.status):
result_code = HTTP_OK
account_info.update(
headers_to_account_info(resp.getheaders()))
break
elif resp.status == HTTP_NOT_FOUND:
if result_code == 0:
result_code = HTTP_NOT_FOUND
elif result_code != HTTP_NOT_FOUND:
result_code = -1
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
continue
else:
result_code = -1
if is_server_error(resp.status):
self.error_occurred(
node,
_('ERROR %(status)d %(body)s From Account '
'Server') %
{'status': resp.status, 'body': body[:1024]})
except (Exception, Timeout):
self.exception_occurred(node, _('Account'),
_('Trying to get account info for %s')
% path)
if self.app.memcache and result_code in (HTTP_OK, HTTP_NOT_FOUND):
if result_code == HTTP_OK:
cache_timeout = self.app.recheck_account_existence
else:
cache_timeout = self.app.recheck_account_existence * 0.1
account_info.update(status=result_code)
self.app.memcache.set(cache_key,
account_info,
time=cache_timeout)
if result_code == HTTP_OK:
try:
container_count = int(account_info['container_count'])
except ValueError:
container_count = 0
return partition, nodes, container_count
return None, None, None
if req:
env = getattr(req, 'environ', {})
else:
env = {}
info = get_info(self.app, env, account)
if not info:
return None, None, None
if info.get('container_count') is None:
container_count = 0
else:
container_count = int(info['container_count'])
return partition, nodes, container_count
def container_info(self, account, container, req=None):
"""
Get container information and thusly verify container existence.
This will also make a call to account_info to verify that the
account exists.
This will also verify account existence.
:param account: account name for the container
:param container: container name to look up
:param req: caller's HTTP request context object (optional)
:param account_autocreate: whether or not to automatically create the
given account or not (optional, default: False)
:returns: dict containing at least container partition ('partition'),
container nodes ('containers'), container read
acl ('read_acl'), container write acl ('write_acl'),
@ -487,69 +585,19 @@ class Controller(object):
Values are set to None if the container does not exist.
"""
part, nodes = self.app.container_ring.get_nodes(account, container)
path = '/%s/%s' % (account, container)
container_info = {'status': 0, 'read_acl': None,
'write_acl': None, 'sync_key': None,
'count': None, 'bytes': None,
'versions': None, 'partition': None,
'nodes': None}
if self.app.memcache:
cache_key = get_container_memcache_key(account, container)
cache_value = self.app.memcache.get(cache_key)
if isinstance(cache_value, dict):
if 'container_size' in cache_value:
cache_value['count'] = cache_value['container_size']
if is_success(cache_value['status']):
container_info.update(cache_value)
container_info['partition'] = part
container_info['nodes'] = nodes
return container_info
if not self.account_info(account, req)[1]:
return container_info
headers = self.generate_request_headers(req)
for node in self.iter_nodes(self.app.container_ring, part):
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], part, 'HEAD',
path, headers)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
body = resp.read()
if is_success(resp.status):
container_info.update(
headers_to_container_info(resp.getheaders()))
break
elif resp.status == HTTP_NOT_FOUND:
container_info['status'] = HTTP_NOT_FOUND
else:
container_info['status'] = -1
if resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(resp.status):
self.error_occurred(node, _(
'ERROR %(status)d %(body)s From Container '
'Server') %
{'status': resp.status, 'body': body[:1024]})
except (Exception, Timeout):
self.exception_occurred(
node, _('Container'),
_('Trying to get container info for %s') % path)
if self.app.memcache:
if container_info['status'] == HTTP_OK:
self.app.memcache.set(
cache_key, container_info,
time=self.app.recheck_container_existence)
elif container_info['status'] == HTTP_NOT_FOUND:
self.app.memcache.set(
cache_key, container_info,
time=self.app.recheck_container_existence * 0.1)
if container_info['status'] == HTTP_OK:
container_info['partition'] = part
container_info['nodes'] = nodes
return container_info
if req:
env = getattr(req, 'environ', {})
else:
env = {}
info = get_info(self.app, env, account, container)
if not info:
info = headers_to_container_info({}, 0)
info['partition'] = None
info['nodes'] = None
else:
info['partition'] = part
info['nodes'] = nodes
return info
def iter_nodes(self, ring, partition):
"""
@ -595,6 +643,23 @@ class Controller(object):
def _make_request(self, nodes, part, method, path, headers, query,
logger_thread_locals):
"""
Sends an HTTP request to a single node and aggregates the result.
It attempts the primary node, then iterates over the handoff nodes
as needed.
:param nodes: an iterator of the backend server and handoff servers
:param part: the partition number
:param method: the method to send to the backend
:param path: the path to send to the backend
:param headers: a list of dicts, where each dict represents one
backend request that should be made.
:param query: query string to send to the backend.
:param logger_thread_locals: The thread local values to be set on the
self.app.logger to retain transaction
logging information.
:returns: a swob.Response object
"""
self.app.logger.thread_locals = logger_thread_locals
for node in nodes:
try:
@ -624,8 +689,14 @@ class Controller(object):
It attempts the primary nodes concurrently, then iterates over the
handoff nodes as needed.
:param req: a request sent by the client
:param ring: the ring used for finding backend servers
:param part: the partition number
:param method: the method to send to the backend
:param path: the path to send to the backend
:param headers: a list of dicts, where each dict represents one
backend request that should be made.
:param query_string: optional query string to send to the backend
:returns: a swob.Response object
"""
start_nodes = ring.get_part_nodes(part)
@ -675,12 +746,22 @@ class Controller(object):
@public
def GET(self, req):
"""Handler for HTTP GET requests."""
"""
Handler for HTTP GET requests.
:param req: The client request
:returns: the response to the client
"""
return self.GETorHEAD(req)
@public
def HEAD(self, req):
"""Handler for HTTP HEAD requests."""
"""
Handler for HTTP HEAD requests.
:param req: The client request
:returns: the response to the client
"""
return self.GETorHEAD(req)
def _make_app_iter_reader(self, node, source, queue, logger_thread_locals):
@ -761,6 +842,11 @@ class Controller(object):
raise
def close_swift_conn(self, src):
"""
Force close the http connection to the backend.
:param src: the response from the backend
"""
try:
src.swift_conn.close()
except Exception:
@ -780,10 +866,19 @@ class Controller(object):
"""
Indicates whether or not the request made to the backend found
what it was looking for.
:param src: the response from the backend
:returns: True if found, False if not
"""
return is_success(src.status) or is_redirection(src.status)
def autocreate_account(self, account):
def autocreate_account(self, env, account):
"""
Autocreate an account
:param env: the environment of the request leading to this autocreate
:param account: the unquoted account name
"""
partition, nodes = self.app.account_ring.get_nodes(account)
path = '/%s' % account
headers = {'X-Timestamp': normalize_timestamp(time.time()),
@ -794,9 +889,7 @@ class Controller(object):
path, [headers] * len(nodes))
if is_success(resp.status_int):
self.app.logger.info('autocreate account %r' % path)
if self.app.memcache:
self.app.memcache.delete(
get_account_memcache_key(account))
clear_info_cache(self.app, env, account)
else:
self.app.logger.warning('Could not autocreate account %r' % path)
@ -862,6 +955,7 @@ class Controller(object):
{'status': possible_source.status,
'body': bodies[-1][:1024],
'type': server_type})
res = None
if sources:
sources.sort(key=lambda s: source_key(s[0]))
source, node = sources.pop()
@ -884,9 +978,15 @@ class Controller(object):
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
return self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method))
if not res:
res = self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method))
try:
(account, container) = split_path(req.path_info, 1, 2)
_set_info_cache(self.app, req.environ, account, container, res)
except ValueError:
pass
return res
def is_origin_allowed(self, cors_info, origin):
"""
@ -926,7 +1026,8 @@ class Controller(object):
# This is a CORS preflight request so check it's allowed
try:
container_info = \
self.container_info(self.account_name, self.container_name)
self.container_info(self.account_name,
self.container_name, req)
except AttributeError:
# This should only happen for requests to the Account. A future
# change could allow CORS requests to the Account level as well.

36
swift/proxy/controllers/container.py
View File

@ -30,7 +30,7 @@ from swift.common.utils import public, csv_append
from swift.common.constraints import check_metadata, MAX_CONTAINER_NAME_LENGTH
from swift.common.http import HTTP_ACCEPTED
from swift.proxy.controllers.base import Controller, delay_denial, \
get_container_memcache_key, headers_to_container_info, cors_validation
cors_validation, clear_info_cache
from swift.common.swob import HTTPBadRequest, HTTPForbidden, \
HTTPNotFound
@ -75,15 +75,6 @@ class ContainerController(Controller):
self.account_name, self.container_name)
resp = self.GETorHEAD_base(
req, _('Container'), self.app.container_ring, part, req.path_info)
if self.app.memcache:
# set the memcache container size for ratelimiting
cache_key = get_container_memcache_key(self.account_name,
self.container_name)
self.app.memcache.set(
cache_key,
headers_to_container_info(resp.headers, resp.status_int),
time=self.app.recheck_container_existence)
if 'swift.authorize' in req.environ:
req.acl = resp.headers.get('x-container-read')
aresp = req.environ['swift.authorize'](req)
@ -124,11 +115,11 @@ class ContainerController(Controller):
(len(self.container_name), MAX_CONTAINER_NAME_LENGTH)
return resp
account_partition, accounts, container_count = \
self.account_info(self.account_name)
self.account_info(self.account_name, req)
if not accounts and self.app.account_autocreate:
self.autocreate_account(self.account_name)
self.autocreate_account(req.environ, self.account_name)
account_partition, accounts, container_count = \
self.account_info(self.account_name)
self.account_info(self.account_name, req)
if not accounts:
return HTTPNotFound(request=req)
if self.app.max_containers_per_account > 0 and \
@ -142,10 +133,8 @@ class ContainerController(Controller):
self.account_name, self.container_name)
headers = self._backend_requests(req, len(containers),
account_partition, accounts)
if self.app.memcache:
cache_key = get_container_memcache_key(self.account_name,
self.container_name)
self.app.memcache.delete(cache_key)
clear_info_cache(self.app, req.environ,
self.account_name, self.container_name)
resp = self.make_requests(
req, self.app.container_ring,
container_partition, 'PUT', req.path_info, headers)
@ -160,15 +149,14 @@ class ContainerController(Controller):
if error_response:
return error_response
account_partition, accounts, container_count = \
self.account_info(self.account_name)
self.account_info(self.account_name, req)
if not accounts:
return HTTPNotFound(request=req)
container_partition, containers = self.app.container_ring.get_nodes(
self.account_name, self.container_name)
headers = self.generate_request_headers(req, transfer=True)
if self.app.memcache:
self.app.memcache.delete(get_container_memcache_key(
self.account_name, self.container_name))
clear_info_cache(self.app, req.environ,
self.account_name, self.container_name)
resp = self.make_requests(
req, self.app.container_ring, container_partition, 'POST',
req.path_info, [headers] * len(containers))
@ -186,10 +174,8 @@ class ContainerController(Controller):
self.account_name, self.container_name)
headers = self._backend_requests(req, len(containers),
account_partition, accounts)
if self.app.memcache:
cache_key = get_container_memcache_key(self.account_name,
self.container_name)
self.app.memcache.delete(cache_key)
clear_info_cache(self.app, req.environ,
self.account_name, self.container_name)
resp = self.make_requests(
req, self.app.container_ring, container_partition, 'DELETE',
req.path_info, headers)

4
swift/proxy/controllers/obj.py
View File

@ -568,7 +568,7 @@ class ObjectController(Controller):
if error_response:
return error_response
container_info = self.container_info(
self.account_name, self.container_name)
self.account_name, self.container_name, req)
container_partition = container_info['partition']
containers = container_info['nodes']
req.acl = container_info['write_acl']
@ -691,7 +691,7 @@ class ObjectController(Controller):
def PUT(self, req):
"""HTTP PUT request handler."""
container_info = self.container_info(
self.account_name, self.container_name)
self.account_name, self.container_name, req)
container_partition = container_info['partition']
containers = container_info['nodes']
req.acl = container_info['write_acl']

73
test/unit/__init__.py
View File

@ -17,6 +17,78 @@ from eventlet import sleep, Timeout
import logging.handlers
from httplib import HTTPException
class FakeRing(object):
def __init__(self, replicas=3):
# 9 total nodes (6 more past the initial 3) is the cap, no matter if
# this is set higher, or R^2 for R replicas
self.replicas = replicas
self.max_more_nodes = 0
self.devs = {}
def set_replicas(self, replicas):
self.replicas = replicas
self.devs = {}
@property
def replica_count(self):
return self.replicas
def get_part(self, account, container=None, obj=None):
return 1
def get_nodes(self, account, container=None, obj=None):
devs = []
for x in xrange(self.replicas):
devs.append(self.devs.get(x))
if devs[x] is None:
self.devs[x] = devs[x] = \
{'ip': '10.0.0.%s' % x,
'port': 1000 + x,
'device': 'sd' + (chr(ord('a') + x)),
'id': x}
return 1, devs
def get_part_nodes(self, part):
return self.get_nodes('blah')[1]
def get_more_nodes(self, nodes):
# replicas^2 is the true cap
for x in xrange(self.replicas, min(self.replicas + self.max_more_nodes,
self.replicas * self.replicas)):
yield {'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sda'}
class FakeMemcache(object):
def __init__(self):
self.store = {}
def get(self, key):
return self.store.get(key)
def keys(self):
return self.store.keys()
def set(self, key, value, time=0):
self.store[key] = value
return True
def incr(self, key, time=0):
self.store[key] = self.store.setdefault(key, 0) + 1
return self.store[key]
@contextmanager
def soft_lock(self, key, timeout=0, retries=5):
yield True
def delete(self, key):
try:
del self.store[key]
except Exception:
pass
return True
def readuntil2crlfs(fd):
rv = ''
@ -133,6 +205,7 @@ class FakeLogger(object):
def exception(self, *args, **kwargs):
self.log_dict['exception'].append((args, kwargs, str(exc_info()[1])))
print 'FakeLogger Exception: %s' % self.log_dict
# mock out the StatsD logging methods:
increment = _store_in('increment')

6
test/unit/common/middleware/test_account_quotas.py
View File

@ -17,6 +17,9 @@ from swift.common.swob import Request
from swift.common.middleware import account_quotas
from swift.proxy.controllers.base import _get_cache_key, \
headers_to_account_info
class FakeCache(object):
def __init__(self, val):
@ -43,6 +46,9 @@ class FakeApp(object):
self.headers = headers
def __call__(self, env, start_response):
# Cache the account_info (same as a real application)
cache_key, env_key = _get_cache_key('a', None)
env[env_key] = headers_to_account_info(self.headers, 200)
start_response('200 OK', self.headers)
return []

47
test/unit/proxy/controllers/test_account.py Normal file
View File

@ -0,0 +1,47 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import unittest
import swift.proxy.controllers.base
from contextlib import contextmanager
from swift.common.swob import Request
from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_account_info
from test.unit import fake_http_connect, FakeRing, FakeMemcache
class TestAccountController(unittest.TestCase):
def setUp(self):
self.app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(),
container_ring=FakeRing(),
object_ring=FakeRing)
def test_account_info_in_response_env(self):
controller = proxy_server.AccountController(self.app, 'AUTH_bob')
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, body='')):
req = Request.blank('/AUTH_bob', {'PATH_INFO': '/AUTH_bob'})
resp = controller.HEAD(req)
self.assertEqual(2, resp.status_int // 100)
self.assertTrue('swift.account/AUTH_bob' in resp.environ)
self.assertEqual(headers_to_account_info(resp.headers),
resp.environ['swift.account/AUTH_bob'])
if __name__ == '__main__':
unittest.main()

197
test/unit/proxy/controllers/test_base.py
View File

@ -14,31 +14,45 @@
# limitations under the License.
import unittest
import swift.proxy.controllers.base
from mock import patch
from swift.proxy.controllers.base import headers_to_container_info, \
headers_to_account_info, get_container_info, get_container_memcache_key, \
get_account_info, get_account_memcache_key
get_account_info, get_account_memcache_key, _get_cache_key, get_info, \
Controller
from swift.common.swob import Request
from swift.common.utils import split_path
import swift.proxy.controllers.base
from test.unit import FakeLogger, fake_http_connect, FakeRing, FakeMemcache
from swift.proxy import server as proxy_server
FakeResponse_status_int = 201
class FakeResponse(object):
def __init__(self, headers):
def __init__(self, headers, env, account, container):
self.headers = headers
self.status_int = 201
self.status_int = FakeResponse_status_int
self.environ = env
cache_key, env_key = _get_cache_key(account, container)
if container:
info = headers_to_container_info(headers, FakeResponse_status_int)
else:
info = headers_to_account_info(headers, FakeResponse_status_int)
env[env_key] = info
class FakeRequest(object):
def __init__(self, env, method, path, swift_source=None):
(version, account,
container, obj) = split_path(env['PATH_INFO'], 2, 4, True)
def __init__(self, env, path):
self.environ = env
(version, account, container, obj) = split_path(path, 2, 4, True)
self.account = account
self.container = container
stype = container and 'container' or 'account'
self.headers = {'x-%s-object-count' % (stype): 1000,
'x-%s-bytes-used' % (stype): 6666}
def get_response(self, app):
return FakeResponse(self.headers)
return FakeResponse(self.headers, self.environ, self.account,
self.container)
class FakeCache(object):
@ -50,22 +64,150 @@ class FakeCache(object):
class TestFuncs(unittest.TestCase):
def setUp(self):
self.app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(),
container_ring=FakeRing(),
object_ring=FakeRing)
def test_GETorHEAD_base(self):
base = Controller(self.app)
req = Request.blank('/a/c')
with patch('swift.proxy.controllers.base.'
'http_connect', fake_http_connect(200)):
resp = base.GETorHEAD_base(req, 'container', FakeRing(), 'part',
'/a/c')
self.assertTrue('swift.container/a/c' in resp.environ)
self.assertEqual(resp.environ['swift.container/a/c']['status'], 200)
req = Request.blank('/a')
with patch('swift.proxy.controllers.base.'
'http_connect', fake_http_connect(200)):
resp = base.GETorHEAD_base(req, 'account', FakeRing(), 'part',
'/a')
self.assertTrue('swift.account/a' in resp.environ)
self.assertEqual(resp.environ['swift.account/a']['status'], 200)
def test_get_info(self):
global FakeResponse_status_int
# Do a non cached call to account
env = {}
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
info_a = get_info(None, env, 'a')
# Check that you got proper info
self.assertEquals(info_a['status'], 201)
self.assertEquals(info_a['bytes'], 6666)
self.assertEquals(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env, {'swift.account/a': info_a})
# Do an env cached call to account
info_a = get_info(None, env, 'a')
# Check that you got proper info
self.assertEquals(info_a['status'], 201)
self.assertEquals(info_a['bytes'], 6666)
self.assertEquals(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env, {'swift.account/a': info_a})
# This time do env cached call to account and non cached to container
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
info_c = get_info(None, env, 'a', 'c')
# Check that you got proper info
self.assertEquals(info_a['status'], 201)
self.assertEquals(info_c['bytes'], 6666)
self.assertEquals(info_c['object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env['swift.account/a'], info_a)
self.assertEquals(env['swift.container/a/c'], info_c)
# This time do a non cached call to account than non cached to container
env = {} # abandon previous call to env
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
info_c = get_info(None, env, 'a', 'c')
# Check that you got proper info
self.assertEquals(info_a['status'], 201)
self.assertEquals(info_c['bytes'], 6666)
self.assertEquals(info_c['object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env['swift.account/a'], info_a)
self.assertEquals(env['swift.container/a/c'], info_c)
# This time do an env cached call to container while account is not cached
del(env['swift.account/a'])
info_c = get_info(None, env, 'a', 'c')
# Check that you got proper info
self.assertEquals(info_a['status'], 201)
self.assertEquals(info_c['bytes'], 6666)
self.assertEquals(info_c['object_count'], 1000)
# Make sure the env cache is set and account still not cached
self.assertEquals(env, {'swift.container/a/c': info_c})
# Do a non cached call to account not found with ret_not_found
env = {}
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
try:
FakeResponse_status_int = 404
info_a = get_info(None, env, 'a', ret_not_found=True)
finally:
FakeResponse_status_int = 201
# Check that you got proper info
self.assertEquals(info_a['status'], 404)
self.assertEquals(info_a['bytes'], 6666)
self.assertEquals(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env, {'swift.account/a': info_a})
# Do a cached call to account not found with ret_not_found
info_a = get_info(None, env, 'a', ret_not_found=True)
# Check that you got proper info
self.assertEquals(info_a['status'], 404)
self.assertEquals(info_a['bytes'], 6666)
self.assertEquals(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env, {'swift.account/a': info_a})
# Do a non cached call to account not found without ret_not_found
env = {}
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
try:
FakeResponse_status_int = 404
info_a = get_info(None, env, 'a')
finally:
FakeResponse_status_int = 201
# Check that you got proper info
self.assertEquals(info_a, None)
self.assertEquals(env['swift.account/a']['status'], 404)
# Do a cached call to account not found without ret_not_found
info_a = get_info(None, env, 'a')
# Check that you got proper info
self.assertEquals(info_a, None)
self.assertEquals(env['swift.account/a']['status'], 404)
def test_get_container_info_no_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
req = Request.blank("/v1/AUTH_account/cont",
environ={'swift.cache': FakeCache({})})
resp = get_container_info(req.environ, 'xxx')
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_container_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 6666)
self.assertEquals(resp['object_count'], 1000)
def test_get_container_info_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
cached = {'status': 404,
'bytes': 3333,
'object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_container_info(req.environ, 'xxx')
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_container_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3333)
self.assertEquals(resp['object_count'], 10)
self.assertEquals(resp['status'], 404)
@ -80,26 +222,45 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(resp['bytes'], 3867)
def test_get_account_info_no_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
req = Request.blank("/v1/AUTH_account",
environ={'swift.cache': FakeCache({})})
resp = get_account_info(req.environ, 'xxx')
print resp
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 6666)
self.assertEquals(resp['total_object_count'], 1000)
def test_get_account_info_cache(self):
swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
# The original test that we prefer to preserve
cached = {'status': 404,
'bytes': 3333,
'total_object_count': 10}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
resp = get_account_info(req.environ, 'xxx')
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3333)
self.assertEquals(resp['total_object_count'], 10)
self.assertEquals(resp['status'], 404)
# Here is a more realistic test
cached = {'status': 404,
'bytes': '3333',
'container_count': '234',
'total_object_count': '10',
'meta': {}}
req = Request.blank("/v1/account/cont",
environ={'swift.cache': FakeCache(cached)})
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['status'], 404)
self.assertEquals(resp['bytes'], '3333')
self.assertEquals(resp['container_count'], 234)
self.assertEquals(resp['meta'], {})
self.assertEquals(resp['total_object_count'], '10')
def test_get_account_info_env(self):
cache_key = get_account_memcache_key("account")
env_key = 'swift.%s' % cache_key

47
test/unit/proxy/controllers/test_container.py Normal file
View File

@ -0,0 +1,47 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import unittest
import swift.proxy.controllers.base
from contextlib import contextmanager
from swift.common.swob import Request
from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_container_info
from test.unit import fake_http_connect, FakeRing, FakeMemcache
class TestContainerController(unittest.TestCase):
def setUp(self):
self.app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(),
container_ring=FakeRing(),
object_ring=FakeRing)
def test_container_info_in_response_env(self):
controller = proxy_server.ContainerController(self.app, 'a', 'c')
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, body='')):
req = Request.blank('/a/c', {'PATH_INFO': '/a/c'})
resp = controller.HEAD(req)
self.assertEqual(2, resp.status_int // 100)
self.assertTrue("swift.container/a/c" in resp.environ)
self.assertEqual(headers_to_container_info(resp.headers),
resp.environ['swift.container/a/c'])
if __name__ == '__main__':
unittest.main()

264
test/unit/proxy/test_server.py
View File

@ -34,7 +34,8 @@ import mock
from eventlet import sleep, spawn, wsgi, listen
import simplejson
from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect
from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, \
fake_http_connect, FakeRing, FakeMemcache
from swift.proxy import server as proxy_server
from swift.account import server as account_server
from swift.container import server as container_server
@ -195,79 +196,6 @@ def sortHeaderNames(headerNames):
return ', '.join(headers)
class FakeRing(object):
def __init__(self, replicas=3):
# 9 total nodes (6 more past the initial 3) is the cap, no matter if
# this is set higher, or R^2 for R replicas
self.replicas = replicas
self.max_more_nodes = 0
self.devs = {}
def set_replicas(self, replicas):
self.replicas = replicas
self.devs = {}
@property
def replica_count(self):
return self.replicas
def get_part(self, account, container=None, obj=None):
return 1
def get_nodes(self, account, container=None, obj=None):
devs = []
for x in xrange(self.replicas):
devs.append(self.devs.get(x))
if devs[x] is None:
self.devs[x] = devs[x] = \
{'ip': '10.0.0.%s' % x,
'port': 1000 + x,
'device': 'sd' + (chr(ord('a') + x)),
'id': x}
return 1, devs
def get_part_nodes(self, part):
return self.get_nodes('blah')[1]
def get_more_nodes(self, nodes):
# replicas^2 is the true cap
for x in xrange(self.replicas, min(self.replicas + self.max_more_nodes,
self.replicas * self.replicas)):
yield {'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sda'}
class FakeMemcache(object):
def __init__(self):
self.store = {}
def get(self, key):
return self.store.get(key)
def keys(self):
return self.store.keys()
def set(self, key, value, time=0):
self.store[key] = value
return True
def incr(self, key, time=0):
self.store[key] = self.store.setdefault(key, 0) + 1
return self.store[key]
@contextmanager
def soft_lock(self, key, timeout=0, retries=5):
yield True
def delete(self, key):
try:
del self.store[key]
except Exception:
pass
return True
class FakeMemcacheReturnsNone(FakeMemcache):
def get(self, key):
@ -392,9 +320,11 @@ class TestController(unittest.TestCase):
self.check_account_info_return(partition, nodes)
self.assertEquals(count, 12345)
# Test the internal representation in memcache
# 'container_count' changed from int to str
cache_key = get_account_memcache_key(self.account)
container_info = {'status': 200,
'container_count': 12345,
'container_count': '12345',
'total_object_count': None,
'bytes': None,
'meta': {}}
@ -416,9 +346,11 @@ class TestController(unittest.TestCase):
self.check_account_info_return(partition, nodes, True)
self.assertEquals(count, None)
# Test the internal representation in memcache
# 'container_count' changed from 0 to None
cache_key = get_account_memcache_key(self.account)
container_info = {'status': 404,
'container_count': 0,
'container_count': None, # internally keep None
'total_object_count': None,
'bytes': None,
'meta': {}}
@ -442,8 +374,9 @@ class TestController(unittest.TestCase):
self.assertEquals(count, None)
with save_globals():
test(503, 404, 404)
test(404, 404, 503)
# We cache if we have two 404 responses - fail if only one
test(503, 503, 404)
test(504, 404, 503)
test(404, 507, 503)
test(503, 503, 503)
@ -481,14 +414,12 @@ class TestController(unittest.TestCase):
# tests if 200 is cached and used
def test_container_info_200(self):
def account_info(self, account, request, autocreate=False):
return True, True, 0
with save_globals():
headers = {'x-container-read': self.read_acl,
'x-container-write': self.write_acl}
swift.proxy.controllers.Controller.account_info = account_info
set_http_connect(200, headers=headers)
set_http_connect(200, # account_info is found
200, headers=headers) # container_info is found
ret = self.controller.container_info(
self.account, self.container, self.request)
self.check_container_info_return(ret)
@ -506,12 +437,28 @@ class TestController(unittest.TestCase):
# tests if 404 is cached and used
def test_container_info_404(self):
def account_info(self, account, request, autocreate=False):
def account_info(self, account, request):
return True, True, 0
with save_globals():
swift.proxy.controllers.Controller.account_info = account_info
set_http_connect(404, 404, 404)
set_http_connect(503, 204, # account_info found
504, 404, 404) # container_info 'NotFound'
ret = self.controller.container_info(
self.account, self.container, self.request)
self.check_container_info_return(ret, True)
cache_key = get_container_memcache_key(self.account,
self.container)
cache_value = self.memcache.get(cache_key)
self.assertTrue(isinstance(cache_value, dict))
self.assertEquals(404, cache_value.get('status'))
set_http_connect()
ret = self.controller.container_info(
self.account, self.container, self.request)
self.check_container_info_return(ret, True)
set_http_connect(503, 404, 404)# account_info 'NotFound'
ret = self.controller.container_info(
self.account, self.container, self.request)
self.check_container_info_return(ret, True)
@ -537,8 +484,9 @@ class TestController(unittest.TestCase):
self.check_container_info_return(ret, True)
with save_globals():
test(503, 404, 404)
test(404, 404, 503)
# We cache if we have two 404 responses - fail if only one
test(503, 503, 404)
test(504, 404, 503)
test(404, 507, 503)
test(503, 503, 503)
@ -2328,36 +2276,50 @@ class TestObjectController(unittest.TestCase):
set_http_connect(404, 404, 404)
# acct acct acct
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(503, 404, 404)
# acct acct acct
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(503, 503, 404)
# acct acct acct
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(503, 503, 503)
# acct acct acct
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(200, 200, 204, 204, 204)
# acct cont obj obj obj
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 204)
set_http_connect(200, 404, 404, 404)
# acct cont cont cont
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(200, 503, 503, 503)
# acct cont cont cont
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
@ -2367,6 +2329,8 @@ class TestObjectController(unittest.TestCase):
set_http_connect(200)
# acct [isn't actually called since everything
# is error limited]
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
@ -2378,6 +2342,8 @@ class TestObjectController(unittest.TestCase):
set_http_connect(200, 200)
# acct cont [isn't actually called since
# everything is error limited]
# make sure to use a fresh request without cached env
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
@ -4633,10 +4599,10 @@ class TestContainerController(unittest.TestCase):
def test_HEAD_GET(self):
with save_globals():
controller = proxy_server.ContainerController(self.app, 'account',
'container')
controller = proxy_server.ContainerController(self.app, 'a', 'c')
def test_status_map(statuses, expected, **kwargs):
def test_status_map(statuses, expected,
c_expected=None, a_expected=None, **kwargs):
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/a/c', {})
@ -4647,6 +4613,18 @@ class TestContainerController(unittest.TestCase):
if expected < 400:
self.assert_('x-works' in res.headers)
self.assertEquals(res.headers['x-works'], 'yes')
if c_expected:
self.assertTrue('swift.container/a/c' in res.environ)
self.assertEquals(res.environ['swift.container/a/c']['status'],
c_expected)
else:
self.assertTrue('swift.container/a/c' not in res.environ)
if a_expected:
self.assertTrue('swift.account/a' in res.environ)
self.assertEquals(res.environ['swift.account/a']['status'],
a_expected)
else:
self.assertTrue('swift.account/a' not in res.environ)
set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
@ -4658,17 +4636,37 @@ class TestContainerController(unittest.TestCase):
if expected < 400:
self.assert_('x-works' in res.headers)
self.assertEquals(res.headers['x-works'], 'yes')
test_status_map((200, 200, 404, 404), 200)
test_status_map((200, 200, 500, 404), 200)
test_status_map((200, 304, 500, 404), 304)
test_status_map((200, 404, 404, 404), 404)
test_status_map((200, 404, 404, 500), 404)
test_status_map((200, 500, 500, 500), 503)
if c_expected:
self.assertTrue('swift.container/a/c' in res.environ)
self.assertEquals(res.environ['swift.container/a/c']['status'],
c_expected)
else:
self.assertTrue('swift.container/a/c' not in res.environ)
if a_expected:
self.assertTrue('swift.account/a' in res.environ)
self.assertEquals(res.environ['swift.account/a']['status'],
a_expected)
else:
self.assertTrue('swift.account/a' not in res.environ)
# In all the following tests cache 200 for account
# return and ache vary for container
# return 200 and cache 200 for and container
test_status_map((200, 200, 404, 404), 200, 200, 200)
test_status_map((200, 200, 500, 404), 200, 200, 200)
# return 304 dont cache container
test_status_map((200, 304, 500, 404), 304, None, 200)
# return 404 and cache 404 for container
test_status_map((200, 404, 404, 404), 404, 404, 200)
test_status_map((200, 404, 404, 500), 404, 404, 200)
# return 503, dont cache container
test_status_map((200, 500, 500, 500), 503, None, 200)
self.assertFalse(self.app.account_autocreate)
test_status_map((404, 404, 404), 404)
self.app.account_autocreate = True
test_status_map((404, 404, 404), 404)
# In all the following tests cache 404 for account
# return 404 (as account is not found) and dont cache container
test_status_map((404, 404, 404), 404, None, 404)
self.app.account_autocreate = True # This should make no difference
test_status_map((404, 404, 404), 404, None, 404)
def test_PUT(self):
with save_globals():
@ -4821,14 +4819,20 @@ class TestContainerController(unittest.TestCase):
self.assertEquals(resp.status_int, 200)
set_http_connect(404, 404, 404, 200, 200, 200)
# Make sure it is a blank request wthout env caching
req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth})
resp = getattr(controller, meth)(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(503, 404, 404)
# Make sure it is a blank request wthout env caching
req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth})
resp = getattr(controller, meth)(req)
self.assertEquals(resp.status_int, 404)
set_http_connect(503, 404, raise_exc=True)
# Make sure it is a blank request wthout env caching
req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth})
resp = getattr(controller, meth)(req)
self.assertEquals(resp.status_int, 404)
@ -4836,6 +4840,8 @@ class TestContainerController(unittest.TestCase):
dev['errors'] = self.app.error_suppression_limit + 1
dev['last_error'] = time.time()
set_http_connect(200, 200, 200, 200, 200, 200)
# Make sure it is a blank request wthout env caching
req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth})
resp = getattr(controller, meth)(req)
self.assertEquals(resp.status_int, 404)
@ -5132,6 +5138,7 @@ class TestContainerController(unittest.TestCase):
req = Request.blank('/a/c')
self.app.update_request(req)
res = controller.GET(req)
self.assertEquals(res.environ['swift.container/a/c']['status'], 204)
self.assertEquals(res.content_length, 0)
self.assertTrue('transfer-encoding' not in res.headers)
@ -5149,6 +5156,7 @@ class TestContainerController(unittest.TestCase):
req.environ['swift.authorize'] = authorize
self.app.update_request(req)
res = controller.GET(req)
self.assertEquals(res.environ['swift.container/a/c']['status'], 201)
self.assert_(called[0])
def test_HEAD_calls_authorize(self):
@ -5444,18 +5452,24 @@ class TestAccountController(unittest.TestCase):
container_ring=FakeRing(),
object_ring=FakeRing)
def assert_status_map(self, method, statuses, expected):
def assert_status_map(self, method, statuses, expected, env_expected=None):
with save_globals():
set_http_connect(*statuses)
req = Request.blank('/a', {})
self.app.update_request(req)
res = method(req)
self.assertEquals(res.status_int, expected)
if env_expected:
self.assertEquals(res.environ['swift.account/a']['status'],
env_expected)
set_http_connect(*statuses)
req = Request.blank('/a/', {})
self.app.update_request(req)
res = method(req)
self.assertEquals(res.status_int, expected)
if env_expected:
self.assertEquals(res.environ['swift.account/a']['status'],
env_expected)
def test_OPTIONS(self):
with save_globals():
@ -5501,23 +5515,23 @@ class TestAccountController(unittest.TestCase):
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
# GET returns after the first successful call to an Account Server
self.assert_status_map(controller.GET, (200,), 200)
self.assert_status_map(controller.GET, (503, 200), 200)
self.assert_status_map(controller.GET, (503, 503, 200), 200)
self.assert_status_map(controller.GET, (204,), 204)
self.assert_status_map(controller.GET, (503, 204), 204)
self.assert_status_map(controller.GET, (503, 503, 204), 204)
self.assert_status_map(controller.GET, (404, 200), 200)
self.assert_status_map(controller.GET, (404, 404, 200), 200)
self.assert_status_map(controller.GET, (404, 503, 204), 204)
self.assert_status_map(controller.GET, (200,), 200, 200)
self.assert_status_map(controller.GET, (503, 200), 200, 200)
self.assert_status_map(controller.GET, (503, 503, 200), 200, 200)
self.assert_status_map(controller.GET, (204,), 204, 204)
self.assert_status_map(controller.GET, (503, 204), 204, 204)
self.assert_status_map(controller.GET, (503, 503, 204), 204, 204)
self.assert_status_map(controller.GET, (404, 200), 200, 200)
self.assert_status_map(controller.GET, (404, 404, 200), 200, 200)
self.assert_status_map(controller.GET, (404, 503, 204), 204, 204)
# If Account servers fail, if autocreate = False, return majority
# response
self.assert_status_map(controller.GET, (404, 404, 404), 404)
self.assert_status_map(controller.GET, (404, 404, 503), 404)
self.assert_status_map(controller.GET, (404, 404, 404), 404, 404)
self.assert_status_map(controller.GET, (404, 404, 503), 404, 404)
self.assert_status_map(controller.GET, (404, 503, 503), 503)
self.app.memcache = FakeMemcacheReturnsNone()
self.assert_status_map(controller.GET, (404, 404, 404), 404)
self.assert_status_map(controller.GET, (404, 404, 404), 404, 404)
def test_GET_autocreate(self):
@ -5548,19 +5562,19 @@ class TestAccountController(unittest.TestCase):
# Same behaviour as GET
with save_globals():
controller = proxy_server.AccountController(self.app, 'account')
self.assert_status_map(controller.HEAD, (200,), 200)
self.assert_status_map(controller.HEAD, (503, 200), 200)
self.assert_status_map(controller.HEAD, (503, 503, 200), 200)
self.assert_status_map(controller.HEAD, (204,), 204)
self.assert_status_map(controller.HEAD, (503, 204), 204)
self.assert_status_map(controller.HEAD, (204, 503, 503), 204)
self.assert_status_map(controller.HEAD, (204,), 204)
self.assert_status_map(controller.HEAD, (404, 404, 404), 404)
self.assert_status_map(controller.HEAD, (404, 404, 200), 200)
self.assert_status_map(controller.HEAD, (404, 200), 200)
self.assert_status_map(controller.HEAD, (404, 404, 503), 404)
self.assert_status_map(controller.HEAD, (200,), 200, 200)
self.assert_status_map(controller.HEAD, (503, 200), 200, 200)
self.assert_status_map(controller.HEAD, (503, 503, 200), 200, 200)
self.assert_status_map(controller.HEAD, (204,), 204, 204)
self.assert_status_map(controller.HEAD, (503, 204), 204, 204)
self.assert_status_map(controller.HEAD, (204, 503, 503), 204, 204)
self.assert_status_map(controller.HEAD, (204,), 204, 204)
self.assert_status_map(controller.HEAD, (404, 404, 404), 404, 404)
self.assert_status_map(controller.HEAD, (404, 404, 200), 200, 200)
self.assert_status_map(controller.HEAD, (404, 200), 200, 200)
self.assert_status_map(controller.HEAD, (404, 404, 503), 404, 404)
self.assert_status_map(controller.HEAD, (404, 503, 503), 503)
self.assert_status_map(controller.HEAD, (404, 503, 204), 204)
self.assert_status_map(controller.HEAD, (404, 503, 204), 204, 204)
def test_HEAD_autocreate(self):
# Same behaviour as GET