openstack/swift
Code Issues Proposed changes

Merge from trunk

Browse Source
This commit is contained in:
gholt 2011-06-16 22:05:03 +00:00
commit 1dca388dec
12 changed files with 558 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
doc/source/deployment_guide.rst
View File

@ -547,6 +547,16 @@ error_suppression_limit 10 Error count to consider a
node error limited node error limited
allow_account_management false Whether account PUTs and DELETEs allow_account_management false Whether account PUTs and DELETEs
are even callable are even callable
object_post_as_copy true Set object_post_as_copy = false
to turn on fast posts where only
the metadata changes are stored
anew and the original data file
is kept in place. This makes for
quicker posts; but since the
container metadata isn't updated
in this mode, features like
container sync won't be able to
sync posts.
account_autocreate false If set to 'true' authorized account_autocreate false If set to 'true' authorized
accounts that do not yet exist accounts that do not yet exist
within the Swift cluster will within the Swift cluster will

5
etc/proxy-server.conf-sample
View File

@ -40,6 +40,11 @@ use = egg:swift#proxy
# If set to 'true' any authorized user may create and delete accounts; if # If set to 'true' any authorized user may create and delete accounts; if
# 'false' no one, even authorized, can. # 'false' no one, even authorized, can.
# allow_account_management = false # allow_account_management = false
# Set object_post_as_copy = false to turn on fast posts where only the metadata
# changes are stored anew and the original data file is kept in place. This
# makes for quicker posts; but since the container metadata isn't updated in
# this mode, features like container sync won't be able to sync posts.
# object_post_as_copy = true
# If set to 'true' authorized accounts that do not yet exist within the Swift # If set to 'true' authorized accounts that do not yet exist within the Swift
# cluster will be automatically created. # cluster will be automatically created.
# account_autocreate = false # account_autocreate = false

51
swift/common/direct_client.py
View File

@ -36,6 +36,57 @@ def quote(value, safe='/'):
return _quote(value, safe) return _quote(value, safe)
def direct_get_account(node, part, account, marker=None, limit=None,
prefix=None, delimiter=None, conn_timeout=5,
response_timeout=15):
"""
Get listings directly from the account server.
:param node: node dictionary from the ring
:param part: partition the account is on
:param account: account name
:param marker: marker query
:param limit: query limit
:param prefix: prefix query
:param delimeter: delimeter for the query
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: a tuple of (response headers, a list of containers) The response
headers will be a dict and all header names will be lowercase.
"""
path = '/' + account
qs = 'format=json'
if marker:
qs += '&marker=%s' % quote(marker)
if limit:
qs += '&limit=%d' % limit
if prefix:
qs += '&prefix=%s' % quote(prefix)
if delimiter:
qs += '&delimiter=%s' % quote(delimiter)
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'GET', path, query_string='format=json')
with Timeout(response_timeout):
resp = conn.getresponse()
if resp.status < 200 or resp.status >= 300:
resp.read()
raise ClientException(
'Account server %s:%s direct GET %s gave status %s' % (node['ip'],
node['port'], repr('/%s/%s%s' % (node['device'], part, path)),
resp.status),
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status == 204:
resp.read()
return resp_headers, []
return resp_headers, json_loads(resp.read())
def direct_head_container(node, part, account, container, conn_timeout=5, def direct_head_container(node, part, account, container, conn_timeout=5,
response_timeout=15): response_timeout=15):
""" """

2
swift/obj/server.py
View File

@ -627,6 +627,7 @@ class ObjectController(object):
file.keep_cache = True file.keep_cache = True
if 'Content-Encoding' in file.metadata: if 'Content-Encoding' in file.metadata:
response.content_encoding = file.metadata['Content-Encoding'] response.content_encoding = file.metadata['Content-Encoding']
response.headers['X-Timestamp'] = file.metadata['X-Timestamp']
return request.get_response(response) return request.get_response(response)
def HEAD(self, request): def HEAD(self, request):
@ -663,6 +664,7 @@ class ObjectController(object):
response.content_length = file_size response.content_length = file_size
if 'Content-Encoding' in file.metadata: if 'Content-Encoding' in file.metadata:
response.content_encoding = file.metadata['Content-Encoding'] response.content_encoding = file.metadata['Content-Encoding']
response.headers['X-Timestamp'] = file.metadata['X-Timestamp']
return response return response
def DELETE(self, request): def DELETE(self, request):

67
swift/proxy/server.py
View File

@ -162,6 +162,7 @@ class SegmentedIterable(object):
if self.segment > 10: if self.segment > 10:
sleep(max(self.next_get_time - time.time(), 0)) sleep(max(self.next_get_time - time.time(), 0))
self.next_get_time = time.time() + 1 self.next_get_time = time.time() + 1
shuffle(nodes)
resp = self.controller.GETorHEAD_base(req, _('Object'), partition, resp = self.controller.GETorHEAD_base(req, _('Object'), partition,
self.controller.iter_nodes(partition, nodes, self.controller.iter_nodes(partition, nodes,
self.controller.app.object_ring), path, self.controller.app.object_ring), path,
@ -609,6 +610,8 @@ class Controller(object):
statuses = [] statuses = []
reasons = [] reasons = []
bodies = [] bodies = []
source = None
newest = req.headers.get('x-newest', 'f').lower() in TRUE_VALUES
for node in nodes: for node in nodes:
if len(statuses) >= attempts: if len(statuses) >= attempts:
break break
@ -621,23 +624,48 @@ class Controller(object):
headers=req.headers, headers=req.headers,
query_string=req.query_string) query_string=req.query_string)
with Timeout(self.app.node_timeout): with Timeout(self.app.node_timeout):
source = conn.getresponse() possible_source = conn.getresponse()
except (Exception, TimeoutError): except (Exception, TimeoutError):
self.exception_occurred(node, server_type, self.exception_occurred(node, server_type,
_('Trying to %(method)s %(path)s') % _('Trying to %(method)s %(path)s') %
{'method': req.method, 'path': req.path}) {'method': req.method, 'path': req.path})
continue continue
if source.status == 507: if possible_source.status == 507:
self.error_limit(node) self.error_limit(node)
continue continue
if 200 <= source.status <= 399: if 200 <= possible_source.status <= 399:
# 404 if we know we don't have a synced copy # 404 if we know we don't have a synced copy
if not float(source.getheader('X-PUT-Timestamp', '1')): if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
statuses.append(404) statuses.append(404)
reasons.append('') reasons.append('')
bodies.append('') bodies.append('')
source.read() possible_source.read()
continue continue
if (req.method == 'GET' and
possible_source.status in (200, 206)) or \
200 <= possible_source.status <= 399:
if newest:
ts = 0
if source:
ts = float(source.getheader('x-put-timestamp') or
source.getheader('x-timestamp') or 0)
pts = float(possible_source.getheader('x-put-timestamp') or
possible_source.getheader('x-timestamp') or 0)
if pts > ts:
source = possible_source
continue
else:
source = possible_source
break
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append(possible_source.read())
if possible_source.status >= 500:
self.error_occurred(node, _('ERROR %(status)d %(body)s ' \
'From %(type)s Server') %
{'status': possible_source.status,
'body': bodies[-1][:1024], 'type': server_type})
if source:
if req.method == 'GET' and source.status in (200, 206): if req.method == 'GET' and source.status in (200, 206):
res = Response(request=req, conditional_response=True) res = Response(request=req, conditional_response=True)
res.bytes_transferred = 0 res.bytes_transferred = 0
@ -683,13 +711,6 @@ class Controller(object):
res.charset = None res.charset = None
res.content_type = source.getheader('Content-Type') res.content_type = source.getheader('Content-Type')
return res return res
statuses.append(source.status)
reasons.append(source.reason)
bodies.append(source.read())
if source.status >= 500:
self.error_occurred(node, _('ERROR %(status)d %(body)s ' \
'From %(type)s Server') % {'status': source.status,
'body': bodies[-1][:1024], 'type': server_type})
return self.best_response(req, statuses, reasons, bodies, return self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method)) '%s %s' % (server_type, req.method))
@ -744,6 +765,7 @@ class ObjectController(Controller):
lreq = Request.blank('/%s/%s?prefix=%s&format=json&marker=%s' % lreq = Request.blank('/%s/%s?prefix=%s&format=json&marker=%s' %
(quote(self.account_name), quote(lcontainer), (quote(self.account_name), quote(lcontainer),
quote(lprefix), quote(marker))) quote(lprefix), quote(marker)))
shuffle(lnodes)
lresp = self.GETorHEAD_base(lreq, _('Container'), lpartition, lresp = self.GETorHEAD_base(lreq, _('Container'), lpartition,
lnodes, lreq.path_info, lnodes, lreq.path_info,
self.app.container_ring.replica_count) self.app.container_ring.replica_count)
@ -871,6 +893,16 @@ class ObjectController(Controller):
@delay_denial @delay_denial
def POST(self, req): def POST(self, req):
"""HTTP POST request handler.""" """HTTP POST request handler."""
if self.app.object_post_as_copy:
req.method = 'PUT'
req.path_info = '/%s/%s/%s' % (self.account_name,
self.container_name, self.object_name)
req.headers['Content-Length'] = 0
req.headers['X-Copy-From'] = '/%s/%s' % (self.container_name,
self.object_name)
req.headers['X-Fresh-Metadata'] = 'true'
return self.PUT(req)
else:
error_response = check_metadata(req, 'object') error_response = check_metadata(req, 'object')
if error_response: if error_response:
return error_response return error_response
@ -978,6 +1010,7 @@ class ObjectController(Controller):
reader = req.environ['wsgi.input'].read reader = req.environ['wsgi.input'].read
data_source = iter(lambda: reader(self.app.client_chunk_size), '') data_source = iter(lambda: reader(self.app.client_chunk_size), '')
source_header = req.headers.get('X-Copy-From') source_header = req.headers.get('X-Copy-From')
source_resp = None
if source_header: if source_header:
source_header = unquote(source_header) source_header = unquote(source_header)
acct = req.path_info.split('/', 2)[1] acct = req.path_info.split('/', 2)[1]
@ -993,6 +1026,7 @@ class ObjectController(Controller):
'<container name>/<object name>') '<container name>/<object name>')
source_req = req.copy_get() source_req = req.copy_get()
source_req.path_info = source_header source_req.path_info = source_header
source_req.headers['X-Newest'] = 'true'
orig_obj_name = self.object_name orig_obj_name = self.object_name
orig_container_name = self.container_name orig_container_name = self.container_name
self.object_name = src_obj_name self.object_name = src_obj_name
@ -1018,6 +1052,8 @@ class ObjectController(Controller):
if not content_type_manually_set: if not content_type_manually_set:
new_req.headers['Content-Type'] = \ new_req.headers['Content-Type'] = \
source_resp.headers['Content-Type'] source_resp.headers['Content-Type']
if new_req.headers.get('x-fresh-metadata', 'false').lower() \
not in TRUE_VALUES:
for k, v in source_resp.headers.items(): for k, v in source_resp.headers.items():
if k.lower().startswith('x-object-meta-'): if k.lower().startswith('x-object-meta-'):
new_req.headers[k] = v new_req.headers[k] = v
@ -1125,6 +1161,9 @@ class ObjectController(Controller):
if source_header: if source_header:
resp.headers['X-Copied-From'] = quote( resp.headers['X-Copied-From'] = quote(
source_header.split('/', 2)[2]) source_header.split('/', 2)[2])
if 'last-modified' in source_resp.headers:
resp.headers['X-Copied-From-Last-Modified'] = \
source_resp.headers['last-modified']
for k, v in req.headers.items(): for k, v in req.headers.items():
if k.lower().startswith('x-object-meta-'): if k.lower().startswith('x-object-meta-'):
resp.headers[k] = v resp.headers[k] = v
@ -1230,6 +1269,7 @@ class ContainerController(Controller):
return HTTPNotFound(request=req) return HTTPNotFound(request=req)
part, nodes = self.app.container_ring.get_nodes( part, nodes = self.app.container_ring.get_nodes(
self.account_name, self.container_name) self.account_name, self.container_name)
shuffle(nodes)
resp = self.GETorHEAD_base(req, _('Container'), part, nodes, resp = self.GETorHEAD_base(req, _('Container'), part, nodes,
req.path_info, self.app.container_ring.replica_count) req.path_info, self.app.container_ring.replica_count)
@ -1368,6 +1408,7 @@ class AccountController(Controller):
def GETorHEAD(self, req): def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests.""" """Handler for HTTP GET/HEAD requests."""
partition, nodes = self.app.account_ring.get_nodes(self.account_name) partition, nodes = self.app.account_ring.get_nodes(self.account_name)
shuffle(nodes)
resp = self.GETorHEAD_base(req, _('Account'), partition, nodes, resp = self.GETorHEAD_base(req, _('Account'), partition, nodes,
req.path_info.rstrip('/'), self.app.account_ring.replica_count) req.path_info.rstrip('/'), self.app.account_ring.replica_count)
if resp.status_int == 404 and self.app.account_autocreate: if resp.status_int == 404 and self.app.account_autocreate:
@ -1498,6 +1539,8 @@ class BaseApplication(object):
int(conf.get('recheck_account_existence', 60)) int(conf.get('recheck_account_existence', 60))
self.allow_account_management = \ self.allow_account_management = \
conf.get('allow_account_management', 'no').lower() in TRUE_VALUES conf.get('allow_account_management', 'no').lower() in TRUE_VALUES
self.object_post_as_copy = \
conf.get('object_post_as_copy', 'true').lower() in TRUE_VALUES
self.resellers_conf = ConfigParser() self.resellers_conf = ConfigParser()
self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf')) self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf'))
self.object_ring = object_ring or \ self.object_ring = object_ring or \

2
test/functional/swift.py
View File

@ -668,7 +668,7 @@ class File(Base):
self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg)
if self.conn.response.status != 202: if self.conn.response.status not in (201, 202):
raise ResponseError(self.conn.response) raise ResponseError(self.conn.response)
return True return True

4
test/functional/tests.py
View File

@ -1032,7 +1032,7 @@ class TestFile(Base):
self.assert_(file.write()) self.assert_(file.write())
self.assert_status(201) self.assert_status(201)
self.assert_(file.sync_metadata()) self.assert_(file.sync_metadata())
self.assert_status(202) self.assert_status((201, 202))
else: else:
self.assertRaises(ResponseError, file.write) self.assertRaises(ResponseError, file.write)
self.assert_status(400) self.assert_status(400)
@ -1245,7 +1245,7 @@ class TestFile(Base):
file.metadata = metadata file.metadata = metadata
self.assert_(file.sync_metadata()) self.assert_(file.sync_metadata())
self.assert_status(202) self.assert_status((201, 202))
file = self.env.container.file(file.name) file = self.env.container.file(file.name)
self.assert_(file.initialize()) self.assert_(file.initialize())

8
test/probe/test_account_failures.py
View File

@ -20,7 +20,7 @@ from signal import SIGTERM
from subprocess import Popen from subprocess import Popen
from time import sleep from time import sleep
from swift.common import client from swift.common import client, direct_client
from test.probe.common import get_to_final_state, kill_pids, reset_environment from test.probe.common import get_to_final_state, kill_pids, reset_environment
@ -146,7 +146,8 @@ class TestAccountFailures(unittest.TestCase):
sleep(2) sleep(2)
# This is the earlier counts and bytes because the first node doesn't # This is the earlier counts and bytes because the first node doesn't
# have the newest udpates yet. # have the newest udpates yet.
headers, containers = client.get_account(self.url, self.token) headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4') self.assertEquals(headers['x-account-bytes-used'], '4')
@ -167,7 +168,8 @@ class TestAccountFailures(unittest.TestCase):
self.assert_(found2) self.assert_(found2)
get_to_final_state() get_to_final_state()
headers, containers = client.get_account(self.url, self.token) headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9') self.assertEquals(headers['x-account-bytes-used'], '9')

23
test/probe/test_container_failures.py
View File

@ -24,7 +24,7 @@ from uuid import uuid4
import eventlet import eventlet
import sqlite3 import sqlite3
from swift.common import client from swift.common import client, direct_client
from swift.common.utils import hash_path, readconf from swift.common.utils import hash_path, readconf
from test.probe.common import get_to_final_state, kill_pids, reset_environment from test.probe.common import get_to_final_state, kill_pids, reset_environment
@ -72,7 +72,8 @@ class TestContainerFailures(unittest.TestCase):
# This okay because the first node hasn't got the update that the # This okay because the first node hasn't got the update that the
# object was deleted yet. # object was deleted yet.
self.assert_(object1 in [o['name'] for o in self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]) direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# Unfortunately, the following might pass or fail, depending on the # Unfortunately, the following might pass or fail, depending on the
# position of the account server associated with the first container # position of the account server associated with the first container
@ -88,7 +89,8 @@ class TestContainerFailures(unittest.TestCase):
client.put_object(self.url, self.token, container, object2, 'test') client.put_object(self.url, self.token, container, object2, 'test')
# First node still doesn't know object1 was deleted yet; this is okay. # First node still doesn't know object1 was deleted yet; this is okay.
self.assert_(object1 in [o['name'] for o in self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]) direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# And, of course, our new object2 exists. # And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]) client.get_container(self.url, self.token, container)[1]])
@ -150,7 +152,8 @@ class TestContainerFailures(unittest.TestCase):
# server has to indicate the container exists for the put to continue. # server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test') client.put_object(self.url, self.token, container, object2, 'test')
self.assert_(object1 not in [o['name'] for o in self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]) direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# And, of course, our new object2 exists. # And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]) client.get_container(self.url, self.token, container)[1]])
@ -201,7 +204,8 @@ class TestContainerFailures(unittest.TestCase):
# This okay because the first node hasn't got the update that the # This okay because the first node hasn't got the update that the
# object was deleted yet. # object was deleted yet.
self.assert_(object1 in [o['name'] for o in self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]) direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# This fails because all three nodes have to indicate deletion before # This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got # we tell the user it worked. Since the first node 409s (it hasn't got
@ -228,7 +232,8 @@ class TestContainerFailures(unittest.TestCase):
client.put_object(self.url, self.token, container, object2, 'test') client.put_object(self.url, self.token, container, object2, 'test')
# First node still doesn't know object1 was deleted yet; this is okay. # First node still doesn't know object1 was deleted yet; this is okay.
self.assert_(object1 in [o['name'] for o in self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]) direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# And, of course, our new object2 exists. # And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]) client.get_container(self.url, self.token, container)[1]])
@ -277,7 +282,8 @@ class TestContainerFailures(unittest.TestCase):
self.assert_(container in [c['name'] for c in self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]]) client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]) direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# This fails because all three nodes have to indicate deletion before # This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got # we tell the user it worked. Since the first node 409s (it hasn't got
@ -303,7 +309,8 @@ class TestContainerFailures(unittest.TestCase):
# server has to indicate the container exists for the put to continue. # server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test') client.put_object(self.url, self.token, container, object2, 'test')
self.assert_(object1 not in [o['name'] for o in self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]) direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# And, of course, our new object2 exists. # And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]) client.get_container(self.url, self.token, container)[1]])

84
test/probe/test_object_handoff.py
View File

@ -124,47 +124,49 @@ class TestObjectHandoff(unittest.TestCase):
if not exc: if not exc:
raise Exception('Handoff object server still had test object') raise Exception('Handoff object server still had test object')
kill(self.pids[self.port2server[onode['port']]], SIGTERM) # Because POST has changed to a COPY by default, POSTs will succeed on all up
client.post_object(self.url, self.token, container, obj, # nodes now if at least one up node has the object.
headers={'x-object-meta-probe': 'value'}) # kill(self.pids[self.port2server[onode['port']]], SIGTERM)
oheaders = client.head_object(self.url, self.token, container, obj) # client.post_object(self.url, self.token, container, obj,
if oheaders.get('x-object-meta-probe') != 'value': # headers={'x-object-meta-probe': 'value'})
raise Exception('Metadata incorrect, was %s' % repr(oheaders)) # oheaders = client.head_object(self.url, self.token, container, obj)
exc = False # if oheaders.get('x-object-meta-probe') != 'value':
try: # raise Exception('Metadata incorrect, was %s' % repr(oheaders))
direct_client.direct_get_object(another_onode, opart, self.account, # exc = False
container, obj) # try:
except Exception: # direct_client.direct_get_object(another_onode, opart, self.account,
exc = True # container, obj)
if not exc: # except Exception:
raise Exception('Handoff server claimed it had the object when ' # exc = True
'it should not have it') # if not exc:
self.pids[self.port2server[onode['port']]] = Popen([ # raise Exception('Handoff server claimed it had the object when '
'swift-object-server', # 'it should not have it')
'/etc/swift/object-server/%d.conf' % # self.pids[self.port2server[onode['port']]] = Popen([
((onode['port'] - 6000) / 10)]).pid # 'swift-object-server',
sleep(2) # '/etc/swift/object-server/%d.conf' %
oheaders = direct_client.direct_get_object(onode, opart, self.account, # ((onode['port'] - 6000) / 10)]).pid
container, obj)[0] # sleep(2)
if oheaders.get('x-object-meta-probe') == 'value': # oheaders = direct_client.direct_get_object(onode, opart, self.account,
raise Exception('Previously downed object server had the new ' # container, obj)[0]
'metadata when it should not have it') # if oheaders.get('x-object-meta-probe') == 'value':
# Run the extra server last so it'll remove it's extra partition # raise Exception('Previously downed object server had the new '
ps = [] # 'metadata when it should not have it')
for n in onodes: # # Run the extra server last so it'll remove it's extra partition
ps.append(Popen(['swift-object-replicator', # ps = []
'/etc/swift/object-server/%d.conf' % # for n in onodes:
((n['port'] - 6000) / 10), 'once'])) # ps.append(Popen(['swift-object-replicator',
for p in ps: # '/etc/swift/object-server/%d.conf' %
p.wait() # ((n['port'] - 6000) / 10), 'once']))
call(['swift-object-replicator', # for p in ps:
'/etc/swift/object-server/%d.conf' % # p.wait()
((another_onode['port'] - 6000) / 10), 'once']) # call(['swift-object-replicator',
oheaders = direct_client.direct_get_object(onode, opart, self.account, # '/etc/swift/object-server/%d.conf' %
container, obj)[0] # ((another_onode['port'] - 6000) / 10), 'once'])
if oheaders.get('x-object-meta-probe') != 'value': # oheaders = direct_client.direct_get_object(onode, opart, self.account,
raise Exception( # container, obj)[0]
'Previously downed object server did not have the new metadata') # if oheaders.get('x-object-meta-probe') != 'value':
# raise Exception(
# 'Previously downed object server did not have the new metadata')
kill(self.pids[self.port2server[onode['port']]], SIGTERM) kill(self.pids[self.port2server[onode['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, obj) client.delete_object(self.url, self.token, container, obj)

140
test/unit/common/middleware/test_ratelimit.py
View File

@ -15,6 +15,7 @@
import unittest import unittest
import time import time
import eventlet
from contextlib import contextmanager from contextlib import contextmanager
from threading import Thread from threading import Thread
from webob import Request from webob import Request
@ -30,6 +31,7 @@ class FakeMemcache(object):
def __init__(self): def __init__(self):
self.store = {} self.store = {}
self.error_on_incr = False self.error_on_incr = False
self.init_incr_return_neg = False
def get(self, key): def get(self, key):
return self.store.get(key) return self.store.get(key)
@ -41,6 +43,10 @@ class FakeMemcache(object):
def incr(self, key, delta=1, timeout=0): def incr(self, key, delta=1, timeout=0):
if self.error_on_incr: if self.error_on_incr:
raise MemcacheConnectionError('Memcache restarting') raise MemcacheConnectionError('Memcache restarting')
if self.init_incr_return_neg:
# simulate initial hit, force reset of memcache
self.init_incr_return_neg = False
return -10000000
self.store[key] = int(self.store.setdefault(key, 0)) + int(delta) self.store[key] = int(self.store.setdefault(key, 0)) + int(delta)
if self.store[key] < 0: if self.store[key] < 0:
self.store[key] = 0 self.store[key] = 0
@ -109,23 +115,53 @@ def dummy_filter_factory(global_conf, **local_conf):
return ratelimit.RateLimitMiddleware(app, conf, logger=FakeLogger()) return ratelimit.RateLimitMiddleware(app, conf, logger=FakeLogger())
return limit_filter return limit_filter
time_ticker = 0
time_override = []
def mock_sleep(x):
global time_ticker
time_ticker += x
def mock_time():
global time_override
global time_ticker
if time_override:
cur_time = time_override.pop(0)
if cur_time is None:
time_override = [None if i is None else i + time_ticker
for i in time_override]
return time_ticker
return cur_time
return time_ticker
class TestRateLimit(unittest.TestCase): class TestRateLimit(unittest.TestCase):
def _run(self, callable_func, num, rate, extra_sleep=0, def setUp(self):
total_time=None, check_time=True): global time_ticker
time_ticker = 0
self.was_sleep = eventlet.sleep
eventlet.sleep = mock_sleep
self.was_time = time.time
time.time = mock_time
def tearDown(self):
eventlet.sleep = self.was_sleep
time.time = self.was_time
def _run(self, callable_func, num, rate, check_time=True):
global time_ticker
begin = time.time() begin = time.time()
for x in range(0, num): for x in range(0, num):
result = callable_func() result = callable_func()
# Extra sleep is here to test with different call intervals.
time.sleep(extra_sleep)
end = time.time() end = time.time()
if total_time is None: total_time = float(num) / rate - 1.0 / rate # 1st request isn't limited
total_time = num / rate
# Allow for one second of variation in the total time. # Allow for one second of variation in the total time.
time_diff = abs(total_time - (end - begin)) time_diff = abs(total_time - (end - begin))
if check_time: if check_time:
self.assertTrue(time_diff < 1) self.assertEquals(round(total_time, 1), round(time_ticker, 1))
return time_diff return time_diff
def test_get_container_maxrate(self): def test_get_container_maxrate(self):
@ -163,8 +199,8 @@ class TestRateLimit(unittest.TestCase):
'PUT', 'a', 'c', 'o')), 1) 'PUT', 'a', 'c', 'o')), 1)
def test_ratelimit(self): def test_ratelimit(self):
current_rate = 13 current_rate = 5
num_calls = 5 num_calls = 50
conf_dict = {'account_ratelimit': current_rate} conf_dict = {'account_ratelimit': current_rate}
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
ratelimit.http_connect = mock_http_connect(204) ratelimit.http_connect = mock_http_connect(204)
@ -172,9 +208,27 @@ class TestRateLimit(unittest.TestCase):
req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'] = FakeMemcache()
make_app_call = lambda: self.test_ratelimit(req.environ, make_app_call = lambda: self.test_ratelimit(req.environ,
start_response) start_response)
begin = time.time()
self._run(make_app_call, num_calls, current_rate) self._run(make_app_call, num_calls, current_rate)
self.assertEquals(round(time.time() - begin, 1), 9.8)
def test_ratelimit_set_incr(self):
current_rate = 5
num_calls = 50
conf_dict = {'account_ratelimit': current_rate}
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
ratelimit.http_connect = mock_http_connect(204)
req = Request.blank('/v/a')
req.environ['swift.cache'] = FakeMemcache()
req.environ['swift.cache'].init_incr_return_neg = True
make_app_call = lambda: self.test_ratelimit(req.environ,
start_response)
begin = time.time()
self._run(make_app_call, num_calls, current_rate, check_time=False)
self.assertEquals(round(time.time() - begin, 1), 9.8)
def test_ratelimit_whitelist(self): def test_ratelimit_whitelist(self):
global time_ticker
current_rate = 2 current_rate = 2
conf_dict = {'account_ratelimit': current_rate, conf_dict = {'account_ratelimit': current_rate,
'max_sleep_time_seconds': 2, 'max_sleep_time_seconds': 2,
@ -195,7 +249,6 @@ class TestRateLimit(unittest.TestCase):
self.result = self.parent.test_ratelimit(req.environ, self.result = self.parent.test_ratelimit(req.environ,
start_response) start_response)
nt = 5 nt = 5
begin = time.time()
threads = [] threads = []
for i in range(nt): for i in range(nt):
rc = rate_caller(self) rc = rate_caller(self)
@ -206,10 +259,10 @@ class TestRateLimit(unittest.TestCase):
the_498s = [t for t in threads if \ the_498s = [t for t in threads if \
''.join(t.result).startswith('Slow down')] ''.join(t.result).startswith('Slow down')]
self.assertEquals(len(the_498s), 0) self.assertEquals(len(the_498s), 0)
time_took = time.time() - begin self.assertEquals(time_ticker, 0)
self.assert_(time_took < 1)
def test_ratelimit_blacklist(self): def test_ratelimit_blacklist(self):
global time_ticker
current_rate = 2 current_rate = 2
conf_dict = {'account_ratelimit': current_rate, conf_dict = {'account_ratelimit': current_rate,
'max_sleep_time_seconds': 2, 'max_sleep_time_seconds': 2,
@ -231,7 +284,6 @@ class TestRateLimit(unittest.TestCase):
self.result = self.parent.test_ratelimit(req.environ, self.result = self.parent.test_ratelimit(req.environ,
start_response) start_response)
nt = 5 nt = 5
begin = time.time()
threads = [] threads = []
for i in range(nt): for i in range(nt):
rc = rate_caller(self) rc = rate_caller(self)
@ -242,49 +294,35 @@ class TestRateLimit(unittest.TestCase):
the_497s = [t for t in threads if \ the_497s = [t for t in threads if \
''.join(t.result).startswith('Your account')] ''.join(t.result).startswith('Your account')]
self.assertEquals(len(the_497s), 5) self.assertEquals(len(the_497s), 5)
time_took = time.time() - begin self.assertEquals(time_ticker, 0)
self.assert_(round(time_took, 1) == 0)
def test_ratelimit_max_rate_double(self): def test_ratelimit_max_rate_double(self):
global time_ticker
global time_override
current_rate = 2 current_rate = 2
conf_dict = {'account_ratelimit': current_rate, conf_dict = {'account_ratelimit': current_rate,
'clock_accuracy': 100, 'clock_accuracy': 100,
'max_sleep_time_seconds': 1} 'max_sleep_time_seconds': 1}
# making clock less accurate for nosetests running slow
self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp())
ratelimit.http_connect = mock_http_connect(204) ratelimit.http_connect = mock_http_connect(204)
self.test_ratelimit.log_sleep_time_seconds = .00001 self.test_ratelimit.log_sleep_time_seconds = .00001
req = Request.blank('/v/a') req = Request.blank('/v/a')
req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'] = FakeMemcache()
begin = time.time()
class rate_caller(Thread): time_override = [0, 0, 0, 0, None]
# simulates 4 requests coming in at same time, then sleeping
def __init__(self, parent, name): r = self.test_ratelimit(req.environ, start_response)
Thread.__init__(self) mock_sleep(.1)
self.parent = parent r = self.test_ratelimit(req.environ, start_response)
self.name = name mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
def run(self): self.assertEquals(r[0], 'Slow down')
self.result1 = self.parent.test_ratelimit(req.environ, mock_sleep(.1)
start_response) r = self.test_ratelimit(req.environ, start_response)
time.sleep(.1) self.assertEquals(r[0], 'Slow down')
self.result2 = self.parent.test_ratelimit(req.environ, mock_sleep(.1)
start_response) r = self.test_ratelimit(req.environ, start_response)
nt = 3 self.assertEquals(r[0], '204 No Content')
threads = []
for i in range(nt):
rc = rate_caller(self, "thread %s" % i)
rc.start()
threads.append(rc)
for thread in threads:
thread.join()
all_results = [''.join(t.result1) for t in threads]
all_results += [''.join(t.result2) for t in threads]
the_498s = [t for t in all_results if t.startswith('Slow down')]
self.assertEquals(len(the_498s), 2)
time_took = time.time() - begin
self.assert_(1.5 <= round(time_took, 1) < 1.7, time_took)
def test_ratelimit_max_rate_multiple_acc(self): def test_ratelimit_max_rate_multiple_acc(self):
num_calls = 4 num_calls = 4
@ -319,9 +357,9 @@ class TestRateLimit(unittest.TestCase):
threads.append(rc) threads.append(rc)
for thread in threads: for thread in threads:
thread.join() thread.join()
time_took = time.time() - begin time_took = time.time() - begin
# the all 15 threads still take 1.5 secs self.assertEquals(1.5, round(time_took, 1))
self.assert_(1.5 <= round(time_took, 1) < 1.7)
def test_ratelimit_acc_vrs_container(self): def test_ratelimit_acc_vrs_container(self):
conf_dict = {'clock_accuracy': 1000, conf_dict = {'clock_accuracy': 1000,
@ -354,14 +392,13 @@ class TestRateLimit(unittest.TestCase):
threads.append(rc) threads.append(rc)
for thread in threads: for thread in threads:
thread.join() thread.join()
begin = time.time() begin = time.time()
req.environ['swift.cache'].set(cont_key, {'container_size': 20}) req.environ['swift.cache'].set(cont_key, {'container_size': 20})
begin = time.time() begin = time.time()
threads = [] threads = []
runthreads(threads, 3) runthreads(threads, 3)
time_took = time.time() - begin time_took = time.time() - begin
self.assert_(round(time_took, 1) == .4) self.assertEquals(round(time_took, 1), .4)
def test_call_invalid_path(self): def test_call_invalid_path(self):
env = {'REQUEST_METHOD': 'GET', env = {'REQUEST_METHOD': 'GET',
@ -393,7 +430,10 @@ class TestRateLimit(unittest.TestCase):
req.environ['swift.cache'] = None req.environ['swift.cache'] = None
make_app_call = lambda: self.test_ratelimit(req.environ, make_app_call = lambda: self.test_ratelimit(req.environ,
start_response) start_response)
self._run(make_app_call, num_calls, current_rate) begin = time.time()
self._run(make_app_call, num_calls, current_rate, check_time=False)
time_took = time.time() - begin
self.assertEquals(round(time_took, 1), 0) # no memcache, no limiting
def test_restarting_memcache(self): def test_restarting_memcache(self):
current_rate = 2 current_rate = 2
@ -409,7 +449,7 @@ class TestRateLimit(unittest.TestCase):
begin = time.time() begin = time.time()
self._run(make_app_call, num_calls, current_rate, check_time=False) self._run(make_app_call, num_calls, current_rate, check_time=False)
time_took = time.time() - begin time_took = time.time() - begin
self.assert_(round(time_took, 1) == 0) # no memcache, no limiting self.assertEquals(round(time_took, 1), 0) # no memcache, no limiting
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

279
test/unit/proxy/test_server.py
View File

@ -150,7 +150,7 @@ def fake_http_connect(*code_iter, **kwargs):
class FakeConn(object): class FakeConn(object):
def __init__(self, status, etag=None, body=''): def __init__(self, status, etag=None, body='', timestamp='1'):
self.status = status self.status = status
self.reason = 'Fake' self.reason = 'Fake'
self.host = '1.2.3.4' self.host = '1.2.3.4'
@ -159,6 +159,7 @@ def fake_http_connect(*code_iter, **kwargs):
self.received = 0 self.received = 0
self.etag = etag self.etag = etag
self.body = body self.body = body
self.timestamp = timestamp
def getresponse(self): def getresponse(self):
if kwargs.get('raise_exc'): if kwargs.get('raise_exc'):
@ -173,7 +174,8 @@ def fake_http_connect(*code_iter, **kwargs):
def getheaders(self): def getheaders(self):
headers = {'content-length': len(self.body), headers = {'content-length': len(self.body),
'content-type': 'x-application/test', 'content-type': 'x-application/test',
'x-timestamp': '1', 'x-timestamp': self.timestamp,
'last-modified': self.timestamp,
'x-object-meta-test': 'testing', 'x-object-meta-test': 'testing',
'etag': 'etag':
self.etag or '"68b329da9893e34099c7d8ad5cb9c940"', self.etag or '"68b329da9893e34099c7d8ad5cb9c940"',
@ -209,6 +211,7 @@ def fake_http_connect(*code_iter, **kwargs):
def getheader(self, name, default=None): def getheader(self, name, default=None):
return dict(self.getheaders()).get(name.lower(), default) return dict(self.getheaders()).get(name.lower(), default)
timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter))
etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter))
x = kwargs.get('missing_container', [False] * len(code_iter)) x = kwargs.get('missing_container', [False] * len(code_iter))
if not isinstance(x, (tuple, list)): if not isinstance(x, (tuple, list)):
@ -226,9 +229,11 @@ def fake_http_connect(*code_iter, **kwargs):
kwargs['give_connect'](*args, **ckwargs) kwargs['give_connect'](*args, **ckwargs)
status = code_iter.next() status = code_iter.next()
etag = etag_iter.next() etag = etag_iter.next()
timestamp = timestamps_iter.next()
if status == -1: if status == -1:
raise HTTPException() raise HTTPException()
return FakeConn(status, etag, body=kwargs.get('body', '')) return FakeConn(status, etag, body=kwargs.get('body', ''),
timestamp=timestamp)
return connect return connect
@ -962,6 +967,7 @@ class TestObjectController(unittest.TestCase):
def test_POST(self): def test_POST(self):
with save_globals(): with save_globals():
self.app.object_post_as_copy = False
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object') 'container', 'object')
@ -982,6 +988,28 @@ class TestObjectController(unittest.TestCase):
test_status_map((200, 200, 404, 500, 500), 503) test_status_map((200, 200, 404, 500, 500), 503)
test_status_map((200, 200, 404, 404, 404), 404) test_status_map((200, 200, 404, 404, 404), 404)
def test_POST_as_copy(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected):
proxy_server.http_connect = fake_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar'})
self.app.update_request(req)
res = controller.POST(req)
expected = str(expected)
self.assertEquals(res.status[:len(expected)], expected)
test_status_map((200, 200, 200, 200, 200, 202, 202, 202), 202)
test_status_map((200, 200, 200, 200, 200, 202, 202, 500), 202)
test_status_map((200, 200, 200, 200, 200, 202, 500, 500), 503)
test_status_map((200, 200, 200, 200, 200, 202, 404, 500), 503)
test_status_map((200, 200, 200, 200, 200, 202, 404, 404), 404)
test_status_map((200, 200, 200, 200, 200, 404, 500, 500), 503)
test_status_map((200, 200, 200, 200, 200, 404, 404, 404), 404)
def test_DELETE(self): def test_DELETE(self):
with save_globals(): with save_globals():
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
@ -1028,8 +1056,77 @@ class TestObjectController(unittest.TestCase):
test_status_map((404, 404, 500), 404) test_status_map((404, 404, 500), 404)
test_status_map((500, 500, 500), 503) test_status_map((500, 500, 500), 503)
def test_HEAD_newest(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected, timestamps,
expected_timestamp):
proxy_server.http_connect = \
fake_http_connect(*statuses, timestamps=timestamps)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'})
self.app.update_request(req)
res = controller.HEAD(req)
self.assertEquals(res.status[:len(str(expected))],
str(expected))
self.assertEquals(res.headers.get('last-modified'),
expected_timestamp)
test_status_map((200, 200, 200), 200, ('1', '2', '3'), '3')
test_status_map((200, 200, 200), 200, ('1', '3', '2'), '3')
test_status_map((200, 200, 200), 200, ('1', '3', '1'), '3')
test_status_map((200, 200, 200), 200, ('3', '3', '1'), '3')
def test_GET_newest(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected, timestamps,
expected_timestamp):
proxy_server.http_connect = \
fake_http_connect(*statuses, timestamps=timestamps)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'})
self.app.update_request(req)
res = controller.GET(req)
self.assertEquals(res.status[:len(str(expected))],
str(expected))
self.assertEquals(res.headers.get('last-modified'),
expected_timestamp)
test_status_map((200, 200, 200), 200, ('1', '2', '3'), '3')
test_status_map((200, 200, 200), 200, ('1', '3', '2'), '3')
test_status_map((200, 200, 200), 200, ('1', '3', '1'), '3')
test_status_map((200, 200, 200), 200, ('3', '3', '1'), '3')
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected, timestamps,
expected_timestamp):
proxy_server.http_connect = \
fake_http_connect(*statuses, timestamps=timestamps)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {})
self.app.update_request(req)
res = controller.HEAD(req)
self.assertEquals(res.status[:len(str(expected))],
str(expected))
self.assertEquals(res.headers.get('last-modified'),
expected_timestamp)
test_status_map((200, 200, 200), 200, ('1', '2', '3'), '1')
test_status_map((200, 200, 200), 200, ('1', '3', '2'), '1')
test_status_map((200, 200, 200), 200, ('1', '3', '1'), '1')
test_status_map((200, 200, 200), 200, ('3', '3', '1'), '3')
def test_POST_meta_val_len(self): def test_POST_meta_val_len(self):
with save_globals(): with save_globals():
self.app.object_post_as_copy = False
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object') 'container', 'object')
proxy_server.http_connect = \ proxy_server.http_connect = \
@ -1049,8 +1146,30 @@ class TestObjectController(unittest.TestCase):
res = controller.POST(req) res = controller.POST(req)
self.assertEquals(res.status_int, 400) self.assertEquals(res.status_int, 400)
def test_POST_as_copy_meta_val_len(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
# acct cont objc objc objc obj obj obj
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * 256})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 202)
proxy_server.http_connect = fake_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * 257})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
def test_POST_meta_key_len(self): def test_POST_meta_key_len(self):
with save_globals(): with save_globals():
self.app.object_post_as_copy = False
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object') 'container', 'object')
proxy_server.http_connect = \ proxy_server.http_connect = \
@ -1070,6 +1189,27 @@ class TestObjectController(unittest.TestCase):
res = controller.POST(req) res = controller.POST(req)
self.assertEquals(res.status_int, 400) self.assertEquals(res.status_int, 400)
def test_POST_as_copy_meta_key_len(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
# acct cont objc objc objc obj obj obj
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
('X-Object-Meta-' + 'x' * 128): 'x'})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 202)
proxy_server.http_connect = fake_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
('X-Object-Meta-' + 'x' * 129): 'x'})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
def test_POST_meta_count(self): def test_POST_meta_count(self):
with save_globals(): with save_globals():
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
@ -1344,7 +1484,8 @@ class TestObjectController(unittest.TestCase):
self.assert_status_map(controller.HEAD, (200, 200, 200), 503) self.assert_status_map(controller.HEAD, (200, 200, 200), 503)
self.assert_('last_error' in controller.app.object_ring.devs[0]) self.assert_('last_error' in controller.app.object_ring.devs[0])
self.assert_status_map(controller.PUT, (200, 201, 201, 201), 503) self.assert_status_map(controller.PUT, (200, 201, 201, 201), 503)
self.assert_status_map(controller.POST, (200, 202, 202, 202), 503) self.assert_status_map(controller.POST,
(200, 200, 200, 200, 202, 202, 202), 503)
self.assert_status_map(controller.DELETE, self.assert_status_map(controller.DELETE,
(200, 204, 204, 204), 503) (200, 204, 204, 204), 503)
self.app.error_suppression_interval = -300 self.app.error_suppression_interval = -300
@ -1437,18 +1578,41 @@ class TestObjectController(unittest.TestCase):
def test_PUT_POST_requires_container_exist(self): def test_PUT_POST_requires_container_exist(self):
with save_globals(): with save_globals():
self.app.object_post_as_copy = False
self.app.memcache = FakeMemcacheReturnsNone() self.app.memcache = FakeMemcacheReturnsNone()
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object') 'container', 'object')
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 200, 200, 200) fake_http_connect(200, 404, 404, 404, 200, 200, 200)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'})
self.app.update_request(req) self.app.update_request(req)
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEquals(resp.status_int, 404) self.assertEquals(resp.status_int, 404)
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 200, 200, 200) fake_http_connect(200, 404, 404, 404, 200, 200)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
headers={'Content-Type': 'text/plain'})
self.app.update_request(req)
resp = controller.POST(req)
self.assertEquals(resp.status_int, 404)
def test_PUT_POST_as_copy_requires_container_exist(self):
with save_globals():
self.app.memcache = FakeMemcacheReturnsNone()
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 404, 404, 404, 200, 200, 200)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'})
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 404)
proxy_server.http_connect = \
fake_http_connect(200, 404, 404, 404, 200, 200, 200, 200, 200,
200)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'}, req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
headers={'Content-Type': 'text/plain'}) headers={'Content-Type': 'text/plain'})
self.app.update_request(req) self.app.update_request(req)
@ -1568,8 +1732,10 @@ class TestObjectController(unittest.TestCase):
'X-Copy-From': 'c/o'}) 'X-Copy-From': 'c/o'})
self.app.update_request(req) self.app.update_request(req)
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
# acct cont acct cont objc obj obj obj 201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {} self.app.memcache.store = {}
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201) self.assertEquals(resp.status_int, 201)
@ -1581,8 +1747,8 @@ class TestObjectController(unittest.TestCase):
'X-Copy-From': 'c/o'}) 'X-Copy-From': 'c/o'})
self.app.update_request(req) self.app.update_request(req)
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200) fake_http_connect(200, 200, 200, 200, 200, 200, 200)
# acct cont acct cont objc # acct cont acct cont objc objc objc
self.app.memcache.store = {} self.app.memcache.store = {}
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEquals(resp.status_int, 400) self.assertEquals(resp.status_int, 400)
@ -1593,8 +1759,10 @@ class TestObjectController(unittest.TestCase):
'X-Copy-From': 'c/o/o2'}) 'X-Copy-From': 'c/o/o2'})
req.account = 'a' req.account = 'a'
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
# acct cont acct cont objc obj obj obj 201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {} self.app.memcache.store = {}
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201) self.assertEquals(resp.status_int, 201)
@ -1606,8 +1774,10 @@ class TestObjectController(unittest.TestCase):
'X-Copy-From': 'c/o%20o2'}) 'X-Copy-From': 'c/o%20o2'})
req.account = 'a' req.account = 'a'
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
# acct cont acct cont objc obj obj obj 201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {} self.app.memcache.store = {}
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201) self.assertEquals(resp.status_int, 201)
@ -1619,8 +1789,10 @@ class TestObjectController(unittest.TestCase):
'X-Copy-From': '/c/o'}) 'X-Copy-From': '/c/o'})
self.app.update_request(req) self.app.update_request(req)
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
# acct cont acct cont objc obj obj obj 201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {} self.app.memcache.store = {}
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201) self.assertEquals(resp.status_int, 201)
@ -1631,8 +1803,10 @@ class TestObjectController(unittest.TestCase):
'X-Copy-From': '/c/o/o2'}) 'X-Copy-From': '/c/o/o2'})
req.account = 'a' req.account = 'a'
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
# acct cont acct cont objc obj obj obj 201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {} self.app.memcache.store = {}
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201) self.assertEquals(resp.status_int, 201)
@ -1692,8 +1866,8 @@ class TestObjectController(unittest.TestCase):
'X-Object-Meta-Ours': 'okay'}) 'X-Object-Meta-Ours': 'okay'})
self.app.update_request(req) self.app.update_request(req)
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 201, 201, 201) fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
# acct cont objc obj obj obj # acct cont objc objc objc obj obj obj
self.app.memcache.store = {} self.app.memcache.store = {}
resp = controller.PUT(req) resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201) self.assertEquals(resp.status_int, 201)
@ -1717,8 +1891,10 @@ class TestObjectController(unittest.TestCase):
headers={'Destination': 'c/o'}) headers={'Destination': 'c/o'})
req.account = 'a' req.account = 'a'
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
# acct cont acct cont objc obj obj obj 201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {} self.app.memcache.store = {}
resp = controller.COPY(req) resp = controller.COPY(req)
self.assertEquals(resp.status_int, 201) self.assertEquals(resp.status_int, 201)
@ -1730,8 +1906,10 @@ class TestObjectController(unittest.TestCase):
req.account = 'a' req.account = 'a'
controller.object_name = 'o/o2' controller.object_name = 'o/o2'
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
# acct cont acct cont objc obj obj obj 201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {} self.app.memcache.store = {}
resp = controller.COPY(req) resp = controller.COPY(req)
self.assertEquals(resp.status_int, 201) self.assertEquals(resp.status_int, 201)
@ -1742,8 +1920,10 @@ class TestObjectController(unittest.TestCase):
req.account = 'a' req.account = 'a'
controller.object_name = 'o' controller.object_name = 'o'
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
# acct cont acct cont objc obj obj obj 201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {} self.app.memcache.store = {}
resp = controller.COPY(req) resp = controller.COPY(req)
self.assertEquals(resp.status_int, 201) self.assertEquals(resp.status_int, 201)
@ -1755,8 +1935,10 @@ class TestObjectController(unittest.TestCase):
req.account = 'a' req.account = 'a'
controller.object_name = 'o/o2' controller.object_name = 'o/o2'
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
# acct cont acct cont objc obj obj obj 201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {} self.app.memcache.store = {}
resp = controller.COPY(req) resp = controller.COPY(req)
self.assertEquals(resp.status_int, 201) self.assertEquals(resp.status_int, 201)
@ -1812,8 +1994,8 @@ class TestObjectController(unittest.TestCase):
req.account = 'a' req.account = 'a'
controller.object_name = 'o' controller.object_name = 'o'
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 201, 201, 201) fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
# acct cont objc obj obj obj # acct cont objc objc objc obj obj obj
self.app.memcache.store = {} self.app.memcache.store = {}
resp = controller.COPY(req) resp = controller.COPY(req)
self.assertEquals(resp.status_int, 201) self.assertEquals(resp.status_int, 201)
@ -1821,6 +2003,23 @@ class TestObjectController(unittest.TestCase):
'testing') 'testing')
self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay')
def test_COPY_newest(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
headers={'Destination': '/c/o'})
req.account = 'a'
controller.object_name = 'o'
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201,
timestamps=('1', '1', '1', '3', '2', '4', '4', '4'))
# acct cont objc objc objc obj obj obj
self.app.memcache.store = {}
resp = controller.COPY(req)
self.assertEquals(resp.status_int, 201)
self.assertEquals(resp.headers['x-copied-from-last-modified'],
'3')
def test_chunked_put(self): def test_chunked_put(self):
class ChunkedFile(): class ChunkedFile():
@ -2596,6 +2795,7 @@ class TestObjectController(unittest.TestCase):
called[0] = True called[0] = True
return HTTPUnauthorized(request=req) return HTTPUnauthorized(request=req)
with save_globals(): with save_globals():
self.app.object_post_as_copy = False
proxy_server.http_connect = \ proxy_server.http_connect = \
fake_http_connect(200, 200, 201, 201, 201) fake_http_connect(200, 200, 201, 201, 201)
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
@ -2607,6 +2807,24 @@ class TestObjectController(unittest.TestCase):
res = controller.POST(req) res = controller.POST(req)
self.assert_(called[0]) self.assert_(called[0])
def test_POST_as_copy_calls_authorize(self):
called = [False]
def authorize(req):
called[0] = True
return HTTPUnauthorized(request=req)
with save_globals():
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
headers={'Content-Length': '5'}, body='12345')
req.environ['swift.authorize'] = authorize
self.app.update_request(req)
res = controller.POST(req)
self.assert_(called[0])
def test_PUT_calls_authorize(self): def test_PUT_calls_authorize(self):
called = [False] called = [False]
@ -2814,6 +3032,7 @@ class TestContainerController(unittest.TestCase):
def test_error_limiting(self): def test_error_limiting(self):
with save_globals(): with save_globals():
proxy_server.shuffle = lambda l: None
controller = proxy_server.ContainerController(self.app, 'account', controller = proxy_server.ContainerController(self.app, 'account',
'container') 'container')
self.assert_status_map(controller.HEAD, (200, 503, 200, 200), 200, self.assert_status_map(controller.HEAD, (200, 503, 200, 200), 200,