openstack/swift
Code Issues Proposed changes

additional unit test coverage for container/server.py

Browse Source 
Change-Id: Id510cceb88f062f921450a5d25d7c97b7ca5943b
This commit is contained in:
Leah Klearman 2013-03-20 19:26:45 -07:00
parent a979c8007b
commit 60489cdedd
4 changed files with 401 additions and 179 deletions
swift/container
server.py
test/unit
__init__.py
container
test_server.py
proxy
test_server.py

7
swift/container/server.py

@ -94,7 +94,10 @@ class ContainerController(object):
:param container: container name
:param broker: container DB broker object
:returns: if all the account requests return a 404 error code,
HTTPNotFound response object, otherwise None.
HTTPNotFound response object,
if the account cannot be updated due to a malformed header,
an HTTPBadRequest response object,
otherwise None.
"""
account_hosts = [h.strip() for h in
req.headers.get('X-Account-Host', '').split(',')]
@ -110,7 +113,7 @@ class ContainerController(object):
'"%s" vs "%s"' %
(req.headers.get('X-Account-Host', ''),
req.headers.get('X-Account-Device', ''))))
return
return HTTPBadRequest(req=req)
if account_partition:
updates = zip(account_hosts, account_devices)

126
test/unit/__init__.py

@ -16,6 +16,8 @@ from ConfigParser import MissingSectionHeaderError
from StringIO import StringIO
from swift.common.utils import readconf, config_true_value
from logging import Handler
from hashlib import md5
from eventlet import sleep, spawn, Timeout
import logging.handlers
@ -267,3 +269,127 @@ def mock(update):
setattr(module, attr, value)
for module, attr in deletes:
delattr(module, attr)
def fake_http_connect(*code_iter, **kwargs):
class FakeConn(object):
def __init__(self, status, etag=None, body='', timestamp='1',
expect_status=None):
self.status = status
if expect_status is None:
self.expect_status = self.status
else:
self.expect_status = expect_status
self.reason = 'Fake'
self.host = '1.2.3.4'
self.port = '1234'
self.sent = 0
self.received = 0
self.etag = etag
self.body = body
self.timestamp = timestamp
def getresponse(self):
if kwargs.get('raise_exc'):
raise Exception('test')
if kwargs.get('raise_timeout_exc'):
raise Timeout()
return self
def getexpect(self):
if self.expect_status == -2:
raise HTTPException()
if self.expect_status == -3:
return FakeConn(507)
if self.expect_status == -4:
return FakeConn(201)
return FakeConn(100)
def getheaders(self):
etag = self.etag
if not etag:
if isinstance(self.body, str):
etag = '"' + md5(self.body).hexdigest() + '"'
else:
etag = '"68b329da9893e34099c7d8ad5cb9c940"'
headers = {'content-length': len(self.body),
'content-type': 'x-application/test',
'x-timestamp': self.timestamp,
'last-modified': self.timestamp,
'x-object-meta-test': 'testing',
'etag': etag,
'x-works': 'yes',
'x-account-container-count': kwargs.get('count', 12345)}
if not self.timestamp:
del headers['x-timestamp']
try:
if container_ts_iter.next() is False:
headers['x-container-timestamp'] = '1'
except StopIteration:
pass
if 'slow' in kwargs:
headers['content-length'] = '4'
if 'headers' in kwargs:
headers.update(kwargs['headers'])
return headers.items()
def read(self, amt=None):
if 'slow' in kwargs:
if self.sent < 4:
self.sent += 1
sleep(0.1)
return ' '
rv = self.body[:amt]
self.body = self.body[amt:]
return rv
def send(self, amt=None):
if 'slow' in kwargs:
if self.received < 4:
self.received += 1
sleep(0.1)
def getheader(self, name, default=None):
return dict(self.getheaders()).get(name.lower(), default)
timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter))
etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter))
x = kwargs.get('missing_container', [False] * len(code_iter))
if not isinstance(x, (tuple, list)):
x = [x] * len(code_iter)
container_ts_iter = iter(x)
code_iter = iter(code_iter)
static_body = kwargs.get('body', None)
body_iter = kwargs.get('body_iter', None)
if body_iter:
body_iter = iter(body_iter)
def connect(*args, **ckwargs):
if 'give_content_type' in kwargs:
if len(args) >= 7 and 'Content-Type' in args[6]:
kwargs['give_content_type'](args[6]['Content-Type'])
else:
kwargs['give_content_type']('')
if 'give_connect' in kwargs:
kwargs['give_connect'](*args, **ckwargs)
status = code_iter.next()
if isinstance(status, tuple):
status, expect_status = status
else:
expect_status = status
etag = etag_iter.next()
timestamp = timestamps_iter.next()
if status <= 0:
raise HTTPException()
if body_iter is None:
body = static_body or ''
else:
body = body_iter.next()
return FakeConn(status, etag, body=body, timestamp=timestamp,
expect_status=expect_status)
return connect

299
test/unit/container/test_server.py

@ -15,19 +15,30 @@
import operator
import os
import sys
import unittest
from contextlib import contextmanager
from shutil import rmtree
from StringIO import StringIO
from time import time
from tempfile import mkdtemp
from eventlet import spawn, Timeout, listen
import simplejson
from swift.common.swob import Request
from swift.common.swob import Request, HTTPBadRequest
import swift.container
from swift.container import server as container_server
from swift.common.utils import normalize_timestamp, mkdirs
from test.unit import fake_http_connect
@contextmanager
def save_globals():
orig_http_connect = getattr(swift.container.server, 'http_connect',
None)
try:
yield True
finally:
swift.container.server.http_connect = orig_http_connect
class TestContainerController(unittest.TestCase):
@ -98,9 +109,9 @@ class TestContainerController(unittest.TestCase):
self.assert_(response.status.startswith('204'))
self.assertEquals(int(response.headers['x-container-bytes-used']), 0)
self.assertEquals(int(response.headers['x-container-object-count']), 0)
req2 = Request.blank('/sda1/p/a/c/o', environ=
{'HTTP_X_TIMESTAMP': '1', 'HTTP_X_SIZE': 42,
'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x'})
req2 = Request.blank('/sda1/p/a/c/o', environ={
'HTTP_X_TIMESTAMP': '1', 'HTTP_X_SIZE': 42,
'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x'})
self.controller.PUT(req2)
response = self.controller.HEAD(req)
self.assertEquals(int(response.headers['x-container-bytes-used']), 42)
@ -111,6 +122,29 @@ class TestContainerController(unittest.TestCase):
resp = self.controller.HEAD(req)
self.assertEquals(resp.status_int, 404)
def test_HEAD_invalid_partition(self):
req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'HEAD',
'HTTP_X_TIMESTAMP': '1'})
resp = self.controller.HEAD(req)
self.assertEquals(resp.status_int, 400)
def test_HEAD_insufficient_storage(self):
self.controller = container_server.ContainerController(
{'devices': self.testdir})
req = Request.blank('/sda-null/p/a/c', environ={'REQUEST_METHOD': 'HEAD',
'HTTP_X_TIMESTAMP': '1'})
resp = self.controller.HEAD(req)
self.assertEquals(resp.status_int, 507)
def test_HEAD_invalid_content_type(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
self.controller.PUT(req)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'},
headers={'Accept': 'application/plain'})
resp = self.controller.HEAD(req)
self.assertEquals(resp.status_int, 406)
def test_PUT(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1'})
@ -183,6 +217,29 @@ class TestContainerController(unittest.TestCase):
self.assertEquals(resp.status_int, 204)
self.assert_('x-container-meta-test' not in resp.headers)
def test_PUT_invalid_partition(self):
req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1'})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 400)
def test_PUT_timestamp_not_float(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
self.controller.PUT(req)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': 'not-float'})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 400)
def test_PUT_insufficient_storage(self):
self.controller = container_server.ContainerController(
{'devices': self.testdir})
req = Request.blank('/sda-null/p/a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1'})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 507)
def test_POST_HEAD_metadata(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(1)})
@ -231,6 +288,53 @@ class TestContainerController(unittest.TestCase):
self.assertEquals(resp.status_int, 204)
self.assert_('x-container-meta-test' not in resp.headers)
def test_POST_invalid_partition(self):
req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'POST',
'HTTP_X_TIMESTAMP': '1'})
resp = self.controller.POST(req)
self.assertEquals(resp.status_int, 400)
def test_POST_timestamp_not_float(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
self.controller.PUT(req)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': 'not-float'})
resp = self.controller.POST(req)
self.assertEquals(resp.status_int, 400)
def test_POST_insufficient_storage(self):
self.controller = container_server.ContainerController(
{'devices': self.testdir})
req = Request.blank('/sda-null/p/a/c', environ={'REQUEST_METHOD': 'POST',
'HTTP_X_TIMESTAMP': '1'})
resp = self.controller.POST(req)
self.assertEquals(resp.status_int, 507)
def test_POST_invalid_container_sync_to(self):
self.controller = container_server.ContainerController(
{'devices': self.testdir})
req = Request.blank('/sda-null/p/a/c', environ={'REQUEST_METHOD': 'POST',
'HTTP_X_TIMESTAMP': '1'},
headers={'x-container-sync-to': '192.168.0.1'})
resp = self.controller.POST(req)
self.assertEquals(resp.status_int, 400)
def test_POST_after_DELETE_not_found(self):
req = Request.blank('/sda1/p/a/c',
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': '1'})
self.controller.PUT(req)
req = Request.blank('/sda1/p/a/c',
environ={'REQUEST_METHOD': 'DELETE'},
headers={'X-Timestamp': '2'})
self.controller.DELETE(req)
req = Request.blank('/sda1/p/a/c/',
environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': '3'})
resp = self.controller.POST(req)
self.assertEquals(resp.status_int, 404)
def test_DELETE_obj_not_found(self):
req = Request.blank('/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'DELETE'},
@ -238,16 +342,56 @@ class TestContainerController(unittest.TestCase):
resp = self.controller.DELETE(req)
self.assertEquals(resp.status_int, 404)
def test_DELETE_container_not_found(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 201)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE',
'HTTP_X_TIMESTAMP': '1'})
resp = self.controller.DELETE(req)
self.assertEquals(resp.status_int, 404)
def test_PUT_utf8(self):
snowman = u'\u2603'
container_name = snowman.encode('utf-8')
req = Request.blank('/sda1/p/a/%s'%container_name, environ={'REQUEST_METHOD': 'PUT',
req = Request.blank('/sda1/p/a/%s' % container_name, environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1'})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 201)
def test_account_update_mismatched_host_device(self):
req = Request.blank('/sda1/p/a/c',
environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1'},
headers={'X-Timestamp': '0000000001.00000',
'X-Account-Host': '127.0.0.1:0',
'X-Account-Partition': '123',
'X-Account-Device': 'sda1,sda2'})
broker = self.controller._get_container_broker('sda1', 'p', 'a', 'c')
resp = self.controller.account_update(req, 'a', 'c', broker)
self.assertEquals(resp.status_int, 400)
def test_account_update_account_override_deleted(self):
bindsock = listen(('127.0.0.1', 0))
req = Request.blank('/sda1/p/a/c',
environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1'},
headers={'X-Timestamp': '0000000001.00000',
'X-Account-Host': '%s:%s' % bindsock.getsockname(),
'X-Account-Partition': '123',
'X-Account-Device': 'sda1',
'X-Account-Override-Deleted': 'yes'})
with save_globals():
new_connect = fake_http_connect(200, count=123)
swift.container.server.http_connect = new_connect
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 201)
def test_PUT_account_update(self):
bindsock = listen(('127.0.0.1', 0))
def accept(return_code, expected_timestamp):
try:
with Timeout(3):
@ -270,6 +414,7 @@ class TestContainerController(unittest.TestCase):
except BaseException, err:
return err
return None
req = Request.blank('/sda1/p/a/c',
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': '0000000001.00000',
@ -447,6 +592,7 @@ class TestContainerController(unittest.TestCase):
def test_DELETE_account_update(self):
bindsock = listen(('127.0.0.1', 0))
def accept(return_code, expected_timestamp):
try:
with Timeout(3):
@ -469,6 +615,7 @@ class TestContainerController(unittest.TestCase):
except BaseException, err:
return err
return None
req = Request.blank('/sda1/p/a/c',
environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '1'})
resp = self.controller.PUT(req)
@ -530,6 +677,29 @@ class TestContainerController(unittest.TestCase):
raise Exception(err)
self.assert_(not got_exc)
def test_DELETE_invalid_partition(self):
req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'DELETE',
'HTTP_X_TIMESTAMP': '1'})
resp = self.controller.DELETE(req)
self.assertEquals(resp.status_int, 400)
def test_DELETE_timestamp_not_float(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
self.controller.PUT(req)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'},
headers={'X-Timestamp': 'not-float'})
resp = self.controller.DELETE(req)
self.assertEquals(resp.status_int, 400)
def test_DELETE_insufficient_storage(self):
self.controller = container_server.ContainerController(
{'devices': self.testdir})
req = Request.blank('/sda-null/p/a/c', environ={'REQUEST_METHOD': 'DELETE',
'HTTP_X_TIMESTAMP': '1'})
resp = self.controller.DELETE(req)
self.assertEquals(resp.status_int, 507)
def test_GET_over_limit(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': '0'})
@ -554,8 +724,8 @@ class TestContainerController(unittest.TestCase):
self.assertEquals(eval(resp.body), [])
# fill the container
for i in range(3):
req = Request.blank('/sda1/p/a/jsonc/%s'%i, environ=
{'REQUEST_METHOD': 'PUT',
req = Request.blank('/sda1/p/a/jsonc/%s' % i, environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1',
'HTTP_X_CONTENT_TYPE': 'text/plain',
'HTTP_X_ETAG': 'x',
@ -615,8 +785,8 @@ class TestContainerController(unittest.TestCase):
self.assertEquals(resp.status_int, 204)
# fill the container
for i in range(3):
req = Request.blank('/sda1/p/a/plainc/%s'%i, environ=
{'REQUEST_METHOD': 'PUT',
req = Request.blank('/sda1/p/a/plainc/%s' % i, environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1',
'HTTP_X_CONTENT_TYPE': 'text/plain',
'HTTP_X_ETAG': 'x',
@ -669,13 +839,14 @@ class TestContainerController(unittest.TestCase):
def test_GET_json_last_modified(self):
# make a container
req = Request.blank('/sda1/p/a/jsonc', environ={'REQUEST_METHOD': 'PUT',
req = Request.blank('/sda1/p/a/jsonc', environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
resp = self.controller.PUT(req)
for i, d in [(0, 1.5),
(1, 1.0),]:
req = Request.blank('/sda1/p/a/jsonc/%s'%i, environ=
{'REQUEST_METHOD': 'PUT',
(1, 1.0), ]:
req = Request.blank('/sda1/p/a/jsonc/%s' % i, environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': d,
'HTTP_X_CONTENT_TYPE': 'text/plain',
'HTTP_X_ETAG': 'x',
@ -693,7 +864,7 @@ class TestContainerController(unittest.TestCase):
"hash":"x",
"bytes":0,
"content_type":"text/plain",
"last_modified":"1970-01-01T00:00:01.000000"},]
"last_modified":"1970-01-01T00:00:01.000000"}, ]
req = Request.blank('/sda1/p/a/jsonc?format=json',
environ={'REQUEST_METHOD': 'GET'})
@ -709,8 +880,9 @@ class TestContainerController(unittest.TestCase):
resp = self.controller.PUT(req)
# fill the container
for i in range(3):
req = Request.blank('/sda1/p/a/xmlc/%s'%i, environ=
{'REQUEST_METHOD': 'PUT',
req = Request.blank('/sda1/p/a/xmlc/%s' % i,
environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1',
'HTTP_X_CONTENT_TYPE': 'text/plain',
'HTTP_X_ETAG': 'x',
@ -772,7 +944,7 @@ class TestContainerController(unittest.TestCase):
resp = self.controller.PUT(req)
# fill the container
for i in range(3):
req = Request.blank('/sda1/p/a/c/%s'%i, environ= {'REQUEST_METHOD': 'PUT',
req = Request.blank('/sda1/p/a/c/%s' % i, environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain',
'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0})
resp = self.controller.PUT(req)
@ -781,7 +953,7 @@ class TestContainerController(unittest.TestCase):
req = Request.blank('/sda1/p/a/c?limit=2&marker=1', environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
result = resp.body.split()
self.assertEquals(result, ['2',])
self.assertEquals(result, ['2', ])
def test_weird_content_types(self):
snowman = u'\u2603'
@ -789,7 +961,8 @@ class TestContainerController(unittest.TestCase):
'HTTP_X_TIMESTAMP': '0'})
resp = self.controller.PUT(req)
for i, ctype in enumerate((snowman.encode('utf-8'), 'text/plain; "utf-8"')):
req = Request.blank('/sda1/p/a/c/%s'%i, environ= {'REQUEST_METHOD': 'PUT',
req = Request.blank('/sda1/p/a/c/%s' % i, environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': ctype,
'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0})
resp = self.controller.PUT(req)
@ -822,8 +995,9 @@ class TestContainerController(unittest.TestCase):
resp = self.controller.PUT(req)
# fill the container
for i in range(3):
req = Request.blank('/sda1/p/a/c/%s'%i, environ=
{'REQUEST_METHOD': 'PUT',
req = Request.blank('/sda1/p/a/c/%s' % i,
environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1',
'HTTP_X_CONTENT_TYPE': 'text/plain',
'HTTP_X_ETAG': 'x',
@ -834,15 +1008,16 @@ class TestContainerController(unittest.TestCase):
req = Request.blank('/sda1/p/a/c?limit=2', environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
result = resp.body.split()
self.assertEquals(result, ['0','1'])
self.assertEquals(result, ['0', '1'])
def test_GET_prefix(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
resp = self.controller.PUT(req)
for i in ('a1', 'b1', 'a2', 'b2', 'a3', 'b3'):
req = Request.blank('/sda1/p/a/c/%s'%i, environ=
{'REQUEST_METHOD': 'PUT',
req = Request.blank('/sda1/p/a/c/%s' % i,
environ={
'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1',
'HTTP_X_CONTENT_TYPE': 'text/plain',
'HTTP_X_ETAG': 'x',
@ -851,15 +1026,16 @@ class TestContainerController(unittest.TestCase):
self.assertEquals(resp.status_int, 201)
req = Request.blank('/sda1/p/a/c?prefix=a', environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
self.assertEquals(resp.body.split(), ['a1','a2', 'a3'])
self.assertEquals(resp.body.split(), ['a1', 'a2', 'a3'])
def test_GET_delimiter(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
resp = self.controller.PUT(req)
for i in ('US-TX-A', 'US-TX-B', 'US-OK-A', 'US-OK-B', 'US-UT-A'):
req = Request.blank('/sda1/p/a/c/%s'%i, environ=
{'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1',
req = Request.blank('/sda1/p/a/c/%s' % i,
environ={
'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1',
'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x',
'HTTP_X_SIZE': 0})
resp = self.controller.PUT(req)
@ -868,15 +1044,18 @@ class TestContainerController(unittest.TestCase):
environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
self.assertEquals(simplejson.loads(resp.body),
[{"subdir":"US-OK-"},{"subdir":"US-TX-"},{"subdir":"US-UT-"}])
[{"subdir": "US-OK-"},
{"subdir": "US-TX-"},
{"subdir": "US-UT-"}])
def test_GET_delimiter_xml(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
resp = self.controller.PUT(req)
for i in ('US-TX-A', 'US-TX-B', 'US-OK-A', 'US-OK-B', 'US-UT-A'):
req = Request.blank('/sda1/p/a/c/%s'%i, environ=
{'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1',
req = Request.blank('/sda1/p/a/c/%s' % i,
environ={
'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1',
'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x',
'HTTP_X_SIZE': 0})
resp = self.controller.PUT(req)
@ -894,8 +1073,9 @@ class TestContainerController(unittest.TestCase):
'HTTP_X_TIMESTAMP': '0'})
resp = self.controller.PUT(req)
for i in ('US/TX', 'US/TX/B', 'US/OK', 'US/OK/B', 'US/UT/A'):
req = Request.blank('/sda1/p/a/c/%s'%i, environ=
{'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1',
req = Request.blank('/sda1/p/a/c/%s' % i,
environ={
'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1',
'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x',
'HTTP_X_SIZE': 0})
resp = self.controller.PUT(req)
@ -904,17 +1084,27 @@ class TestContainerController(unittest.TestCase):
environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
self.assertEquals(simplejson.loads(resp.body),
[{"name":"US/OK","hash":"x","bytes":0,"content_type":"text/plain",
[{"name":"US/OK", "hash":"x", "bytes":0, "content_type":"text/plain",
"last_modified":"1970-01-01T00:00:01.000000"},
{"name":"US/TX","hash":"x","bytes":0,"content_type":"text/plain",
{"name":"US/TX", "hash":"x", "bytes":0, "content_type":"text/plain",
"last_modified":"1970-01-01T00:00:01.000000"}])
def test_GET_insufficient_storage(self):
self.controller = container_server.ContainerController(
{'devices': self.testdir})
req = Request.blank('/sda-null/p/a/c', environ={'REQUEST_METHOD': 'GET',
'HTTP_X_TIMESTAMP': '1'})
resp = self.controller.GET(req)
self.assertEquals(resp.status_int, 507)
def test_through_call(self):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
outbuf.writelines(args)
self.controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '/sda1/p/a/c',
@ -937,8 +1127,10 @@ class TestContainerController(unittest.TestCase):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
outbuf.writelines(args)
self.controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '/bob',
@ -957,12 +1149,40 @@ class TestContainerController(unittest.TestCase):
self.assertEquals(errbuf.getvalue(), '')
self.assertEquals(outbuf.getvalue()[:4], '400 ')
def test_through_call_invalid_path_utf8(self):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
outbuf.writelines(args)
self.controller.__call__({'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '\x00',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.0',
'CONTENT_LENGTH': '0',
'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http',
'wsgi.input': inbuf,
'wsgi.errors': errbuf,
'wsgi.multithread': False,
'wsgi.multiprocess': False,
'wsgi.run_once': False},
start_response)
self.assertEquals(errbuf.getvalue(), '')
self.assertEquals(outbuf.getvalue()[:4], '412 ')
def test_invalid_method_doesnt_exist(self):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
outbuf.writelines(args)
self.controller.__call__({'REQUEST_METHOD': 'method_doesnt_exist',
'PATH_INFO': '/sda1/p/a/c'},
start_response)
@ -973,8 +1193,10 @@ class TestContainerController(unittest.TestCase):
inbuf = StringIO()
errbuf = StringIO()
outbuf = StringIO()
def start_response(*args):
outbuf.writelines(args)
self.controller.__call__({'REQUEST_METHOD': '__init__',
'PATH_INFO': '/sda1/p/a/c'},
start_response)
@ -1082,8 +1304,10 @@ class TestContainerController(unittest.TestCase):
def test_updating_multiple_container_servers(self):
http_connect_args = []
def fake_http_connect(ipaddr, port, device, partition, method, path,
headers=None, query_string=None, ssl=False):
class SuccessfulFakeConn(object):
@property
def status(self):
@ -1101,7 +1325,7 @@ class TestContainerController(unittest.TestCase):
'headers': headers, 'query_string': query_string}
http_connect_args.append(
dict((k,v) for k,v in captured_args.iteritems()
dict((k, v) for k, v in captured_args.iteritems()
if v is not None))
req = Request.blank(
@ -1154,4 +1378,3 @@ class TestContainerController(unittest.TestCase):
if __name__ == '__main__':
unittest.main()

148
test/unit/proxy/test_server.py

@ -16,20 +16,16 @@
from __future__ import with_statement
import cPickle as pickle
import logging
from logging.handlers import SysLogHandler
import os
import sys
import unittest
import urlparse
import signal
from ConfigParser import ConfigParser
from contextlib import contextmanager
from cStringIO import StringIO
from gzip import GzipFile
from httplib import HTTPException
from shutil import rmtree
import time
from urllib import unquote, quote
from urllib import quote
from hashlib import md5
from tempfile import mkdtemp
import random
@ -38,7 +34,7 @@ import eventlet
from eventlet import sleep, spawn, Timeout, util, wsgi, listen
import simplejson
from test.unit import connect_tcp, readuntil2crlfs, FakeLogger
from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect
from swift.proxy import server as proxy_server
from swift.account import server as account_server
from swift.container import server as container_server
@ -196,130 +192,6 @@ def sortHeaderNames(headerNames):
return ', '.join(headers)
def fake_http_connect(*code_iter, **kwargs):
class FakeConn(object):
def __init__(self, status, etag=None, body='', timestamp='1',
expect_status=None):
self.status = status
if expect_status is None:
self.expect_status = self.status
else:
self.expect_status = expect_status
self.reason = 'Fake'
self.host = '1.2.3.4'
self.port = '1234'
self.sent = 0
self.received = 0
self.etag = etag
self.body = body
self.timestamp = timestamp
def getresponse(self):
if kwargs.get('raise_exc'):
raise Exception('test')
if kwargs.get('raise_timeout_exc'):
raise Timeout()
return self
def getexpect(self):
if self.expect_status == -2:
raise HTTPException()
if self.expect_status == -3:
return FakeConn(507)
if self.expect_status == -4:
return FakeConn(201)
return FakeConn(100)
def getheaders(self):
etag = self.etag
if not etag:
if isinstance(self.body, str):
etag = '"' + md5(self.body).hexdigest() + '"'
else:
etag = '"68b329da9893e34099c7d8ad5cb9c940"'
headers = {'content-length': len(self.body),
'content-type': 'x-application/test',
'x-timestamp': self.timestamp,
'last-modified': self.timestamp,
'x-object-meta-test': 'testing',
'etag': etag,
'x-works': 'yes',
'x-account-container-count': kwargs.get('count', 12345)}
if not self.timestamp:
del headers['x-timestamp']
try:
if container_ts_iter.next() is False:
headers['x-container-timestamp'] = '1'
except StopIteration:
pass
if 'slow' in kwargs:
headers['content-length'] = '4'
if 'headers' in kwargs:
headers.update(kwargs['headers'])
return headers.items()
def read(self, amt=None):
if 'slow' in kwargs:
if self.sent < 4:
self.sent += 1
sleep(0.1)
return ' '
rv = self.body[:amt]
self.body = self.body[amt:]
return rv
def send(self, amt=None):
if 'slow' in kwargs:
if self.received < 4:
self.received += 1
sleep(0.1)
def getheader(self, name, default=None):
return dict(self.getheaders()).get(name.lower(), default)
timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter))
etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter))
x = kwargs.get('missing_container', [False] * len(code_iter))
if not isinstance(x, (tuple, list)):
x = [x] * len(code_iter)
container_ts_iter = iter(x)
code_iter = iter(code_iter)
static_body = kwargs.get('body', None)
body_iter = kwargs.get('body_iter', None)
if body_iter:
body_iter = iter(body_iter)
def connect(*args, **ckwargs):
if 'give_content_type' in kwargs:
if len(args) >= 7 and 'Content-Type' in args[6]:
kwargs['give_content_type'](args[6]['Content-Type'])
else:
kwargs['give_content_type']('')
if 'give_connect' in kwargs:
kwargs['give_connect'](*args, **ckwargs)
status = code_iter.next()
if isinstance(status, tuple):
status, expect_status = status
else:
expect_status = status
etag = etag_iter.next()
timestamp = timestamps_iter.next()
if status <= 0:
raise HTTPException()
if body_iter is None:
body = static_body or ''
else:
body = body_iter.next()
return FakeConn(status, etag, body=body, timestamp=timestamp,
expect_status=expect_status)
return connect
class FakeRing(object):
def __init__(self, replicas=3):
@ -883,7 +755,7 @@ class TestObjectController(unittest.TestCase):
'X-Storage-Token: t\r\n'
'Content-Length: %s\r\n'
'Content-Type: application/octet-stream\r\n'
'\r\n%s' % (path, str(len(obj)), obj))
'\r\n%s' % (path, str(len(obj)), obj))
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
@ -2435,7 +2307,7 @@ class TestObjectController(unittest.TestCase):
headers={'Content-Length': '0',
'X-Copy-From': '/c/o'})
self.app.update_request(req)
class LargeResponseBody(object):
def __len__(self):
@ -2451,7 +2323,6 @@ class TestObjectController(unittest.TestCase):
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 413)
def test_COPY(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
@ -2581,7 +2452,6 @@ class TestObjectController(unittest.TestCase):
resp = controller.COPY(req)
self.assertEquals(resp.status_int, 413)
def test_COPY_newest(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
@ -3986,7 +3856,7 @@ class TestObjectController(unittest.TestCase):
'X-Auth-Token: t\r\n'
'Content-Length: %s\r\n'
'Content-Type: application/octet-stream\r\n'
'\r\n%s' % (obj_len, 'a' * obj_len))
'\r\n%s' % (obj_len, 'a' * obj_len))
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
@ -4175,6 +4045,7 @@ class TestObjectController(unittest.TestCase):
'X-Container-Host',
'X-Container-Device'])
seen_headers = []
def capture_headers(ipaddr, port, device, partition, method,
path, headers=None, query_string=None):
captured = {}
@ -4188,7 +4059,7 @@ class TestObjectController(unittest.TestCase):
set_http_connect(*connect_args, give_connect=capture_headers,
**kwargs)
resp = controller_call(req)
self.assertEqual(2, resp.status_int // 100) # sanity check
self.assertEqual(2, resp.status_int // 100) # sanity check
# don't care about the account/container HEADs, so chuck
# the first two requests
@ -4323,7 +4194,6 @@ class TestObjectController(unittest.TestCase):
'X-Delete-At-Partition': None,
'X-Delete-At-Device': None}])
def test_PUT_x_delete_at_with_more_container_replicas(self):
self.app.container_ring.set_replicas(4)
self.app.expiring_objects_account = 'expires'
@ -5017,7 +4887,7 @@ class TestContainerController(unittest.TestCase):
set_http_connect(*connect_args, give_connect=capture_headers,
**kwargs)
resp = controller_call(req)
self.assertEqual(2, resp.status_int // 100) # sanity check
self.assertEqual(2, resp.status_int // 100) # sanity check
# don't care about the account HEAD, so throw away the
# first element
@ -5140,7 +5010,7 @@ class TestAccountController(unittest.TestCase):
self.app.allow_account_management = False
controller = proxy_server.AccountController(self.app, 'account')
req = Request.blank('/account', {'REQUEST_METHOD': 'OPTIONS'},
headers = {'Origin': 'http://foo.com',
headers={'Origin': 'http://foo.com',
'Access-Control-Request-Method': 'GET'})
req.content_length = 0
resp = controller.OPTIONS(req)