diff --git a/swift/container/server.py b/swift/container/server.py index 89ed555062..562040d30e 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -94,7 +94,10 @@ class ContainerController(object): :param container: container name :param broker: container DB broker object :returns: if all the account requests return a 404 error code, - HTTPNotFound response object, otherwise None. + HTTPNotFound response object, + if the account cannot be updated due to a malformed header, + an HTTPBadRequest response object, + otherwise None. """ account_hosts = [h.strip() for h in req.headers.get('X-Account-Host', '').split(',')] @@ -110,7 +113,7 @@ class ContainerController(object): '"%s" vs "%s"' % (req.headers.get('X-Account-Host', ''), req.headers.get('X-Account-Device', '')))) - return + return HTTPBadRequest(req=req) if account_partition: updates = zip(account_hosts, account_devices) diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 54af5290bc..424d400011 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -16,6 +16,8 @@ from ConfigParser import MissingSectionHeaderError from StringIO import StringIO from swift.common.utils import readconf, config_true_value from logging import Handler +from hashlib import md5 +from eventlet import sleep, spawn, Timeout import logging.handlers @@ -267,3 +269,127 @@ def mock(update): setattr(module, attr, value) for module, attr in deletes: delattr(module, attr) + + +def fake_http_connect(*code_iter, **kwargs): + + class FakeConn(object): + + def __init__(self, status, etag=None, body='', timestamp='1', + expect_status=None): + self.status = status + if expect_status is None: + self.expect_status = self.status + else: + self.expect_status = expect_status + self.reason = 'Fake' + self.host = '1.2.3.4' + self.port = '1234' + self.sent = 0 + self.received = 0 + self.etag = etag + self.body = body + self.timestamp = timestamp + + def getresponse(self): + if kwargs.get('raise_exc'): + raise Exception('test') + if kwargs.get('raise_timeout_exc'): + raise Timeout() + return self + + def getexpect(self): + if self.expect_status == -2: + raise HTTPException() + if self.expect_status == -3: + return FakeConn(507) + if self.expect_status == -4: + return FakeConn(201) + return FakeConn(100) + + def getheaders(self): + etag = self.etag + if not etag: + if isinstance(self.body, str): + etag = '"' + md5(self.body).hexdigest() + '"' + else: + etag = '"68b329da9893e34099c7d8ad5cb9c940"' + + headers = {'content-length': len(self.body), + 'content-type': 'x-application/test', + 'x-timestamp': self.timestamp, + 'last-modified': self.timestamp, + 'x-object-meta-test': 'testing', + 'etag': etag, + 'x-works': 'yes', + 'x-account-container-count': kwargs.get('count', 12345)} + if not self.timestamp: + del headers['x-timestamp'] + try: + if container_ts_iter.next() is False: + headers['x-container-timestamp'] = '1' + except StopIteration: + pass + if 'slow' in kwargs: + headers['content-length'] = '4' + if 'headers' in kwargs: + headers.update(kwargs['headers']) + return headers.items() + + def read(self, amt=None): + if 'slow' in kwargs: + if self.sent < 4: + self.sent += 1 + sleep(0.1) + return ' ' + rv = self.body[:amt] + self.body = self.body[amt:] + return rv + + def send(self, amt=None): + if 'slow' in kwargs: + if self.received < 4: + self.received += 1 + sleep(0.1) + + def getheader(self, name, default=None): + return dict(self.getheaders()).get(name.lower(), default) + + timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) + etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) + x = kwargs.get('missing_container', [False] * len(code_iter)) + if not isinstance(x, (tuple, list)): + x = [x] * len(code_iter) + container_ts_iter = iter(x) + code_iter = iter(code_iter) + static_body = kwargs.get('body', None) + body_iter = kwargs.get('body_iter', None) + if body_iter: + body_iter = iter(body_iter) + + def connect(*args, **ckwargs): + if 'give_content_type' in kwargs: + if len(args) >= 7 and 'Content-Type' in args[6]: + kwargs['give_content_type'](args[6]['Content-Type']) + else: + kwargs['give_content_type']('') + if 'give_connect' in kwargs: + kwargs['give_connect'](*args, **ckwargs) + status = code_iter.next() + if isinstance(status, tuple): + status, expect_status = status + else: + expect_status = status + etag = etag_iter.next() + timestamp = timestamps_iter.next() + + if status <= 0: + raise HTTPException() + if body_iter is None: + body = static_body or '' + else: + body = body_iter.next() + return FakeConn(status, etag, body=body, timestamp=timestamp, + expect_status=expect_status) + + return connect diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 1a0df73a5b..7b7ed03678 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -15,19 +15,30 @@ import operator import os -import sys import unittest +from contextlib import contextmanager from shutil import rmtree from StringIO import StringIO -from time import time from tempfile import mkdtemp from eventlet import spawn, Timeout, listen import simplejson -from swift.common.swob import Request +from swift.common.swob import Request, HTTPBadRequest +import swift.container from swift.container import server as container_server from swift.common.utils import normalize_timestamp, mkdirs +from test.unit import fake_http_connect + + +@contextmanager +def save_globals(): + orig_http_connect = getattr(swift.container.server, 'http_connect', + None) + try: + yield True + finally: + swift.container.server.http_connect = orig_http_connect class TestContainerController(unittest.TestCase): @@ -98,9 +109,9 @@ class TestContainerController(unittest.TestCase): self.assert_(response.status.startswith('204')) self.assertEquals(int(response.headers['x-container-bytes-used']), 0) self.assertEquals(int(response.headers['x-container-object-count']), 0) - req2 = Request.blank('/sda1/p/a/c/o', environ= - {'HTTP_X_TIMESTAMP': '1', 'HTTP_X_SIZE': 42, - 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x'}) + req2 = Request.blank('/sda1/p/a/c/o', environ={ + 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_SIZE': 42, + 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x'}) self.controller.PUT(req2) response = self.controller.HEAD(req) self.assertEquals(int(response.headers['x-container-bytes-used']), 42) @@ -111,6 +122,29 @@ class TestContainerController(unittest.TestCase): resp = self.controller.HEAD(req) self.assertEquals(resp.status_int, 404) + def test_HEAD_invalid_partition(self): + req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'HEAD', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 400) + + def test_HEAD_insufficient_storage(self): + self.controller = container_server.ContainerController( + {'devices': self.testdir}) + req = Request.blank('/sda-null/p/a/c', environ={'REQUEST_METHOD': 'HEAD', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 507) + + def test_HEAD_invalid_content_type(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}, + headers={'Accept': 'application/plain'}) + resp = self.controller.HEAD(req) + self.assertEquals(resp.status_int, 406) + def test_PUT(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) @@ -183,6 +217,29 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 204) self.assert_('x-container-meta-test' not in resp.headers) + def test_PUT_invalid_partition(self): + req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 400) + + def test_PUT_timestamp_not_float(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': 'not-float'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 400) + + def test_PUT_insufficient_storage(self): + self.controller = container_server.ContainerController( + {'devices': self.testdir}) + req = Request.blank('/sda-null/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 507) + def test_POST_HEAD_metadata(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(1)}) @@ -231,6 +288,53 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 204) self.assert_('x-container-meta-test' not in resp.headers) + def test_POST_invalid_partition(self): + req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'POST', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 400) + + def test_POST_timestamp_not_float(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': 'not-float'}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 400) + + def test_POST_insufficient_storage(self): + self.controller = container_server.ContainerController( + {'devices': self.testdir}) + req = Request.blank('/sda-null/p/a/c', environ={'REQUEST_METHOD': 'POST', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 507) + + def test_POST_invalid_container_sync_to(self): + self.controller = container_server.ContainerController( + {'devices': self.testdir}) + req = Request.blank('/sda-null/p/a/c', environ={'REQUEST_METHOD': 'POST', + 'HTTP_X_TIMESTAMP': '1'}, + headers={'x-container-sync-to': '192.168.0.1'}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 400) + + def test_POST_after_DELETE_not_found(self): + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': '1'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': '2'}) + self.controller.DELETE(req) + req = Request.blank('/sda1/p/a/c/', + environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': '3'}) + resp = self.controller.POST(req) + self.assertEquals(resp.status_int, 404) + def test_DELETE_obj_not_found(self): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, @@ -238,16 +342,56 @@ class TestContainerController(unittest.TestCase): resp = self.controller.DELETE(req) self.assertEquals(resp.status_int, 404) + def test_DELETE_container_not_found(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 404) + def test_PUT_utf8(self): snowman = u'\u2603' container_name = snowman.encode('utf-8') - req = Request.blank('/sda1/p/a/%s'%container_name, environ={'REQUEST_METHOD': 'PUT', + req = Request.blank('/sda1/p/a/%s' % container_name, environ={ + 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) resp = self.controller.PUT(req) self.assertEquals(resp.status_int, 201) + def test_account_update_mismatched_host_device(self): + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1'}, + headers={'X-Timestamp': '0000000001.00000', + 'X-Account-Host': '127.0.0.1:0', + 'X-Account-Partition': '123', + 'X-Account-Device': 'sda1,sda2'}) + broker = self.controller._get_container_broker('sda1', 'p', 'a', 'c') + resp = self.controller.account_update(req, 'a', 'c', broker) + self.assertEquals(resp.status_int, 400) + + def test_account_update_account_override_deleted(self): + bindsock = listen(('127.0.0.1', 0)) + req = Request.blank('/sda1/p/a/c', + environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1'}, + headers={'X-Timestamp': '0000000001.00000', + 'X-Account-Host': '%s:%s' % bindsock.getsockname(), + 'X-Account-Partition': '123', + 'X-Account-Device': 'sda1', + 'X-Account-Override-Deleted': 'yes'}) + with save_globals(): + new_connect = fake_http_connect(200, count=123) + swift.container.server.http_connect = new_connect + resp = self.controller.PUT(req) + self.assertEquals(resp.status_int, 201) + def test_PUT_account_update(self): bindsock = listen(('127.0.0.1', 0)) + def accept(return_code, expected_timestamp): try: with Timeout(3): @@ -270,6 +414,7 @@ class TestContainerController(unittest.TestCase): except BaseException, err: return err return None + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '0000000001.00000', @@ -447,6 +592,7 @@ class TestContainerController(unittest.TestCase): def test_DELETE_account_update(self): bindsock = listen(('127.0.0.1', 0)) + def accept(return_code, expected_timestamp): try: with Timeout(3): @@ -469,6 +615,7 @@ class TestContainerController(unittest.TestCase): except BaseException, err: return err return None + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '1'}) resp = self.controller.PUT(req) @@ -530,6 +677,29 @@ class TestContainerController(unittest.TestCase): raise Exception(err) self.assert_(not got_exc) + def test_DELETE_invalid_partition(self): + req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 400) + + def test_DELETE_timestamp_not_float(self): + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': 'not-float'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 400) + + def test_DELETE_insufficient_storage(self): + self.controller = container_server.ContainerController( + {'devices': self.testdir}) + req = Request.blank('/sda-null/p/a/c', environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.DELETE(req) + self.assertEquals(resp.status_int, 507) + def test_GET_over_limit(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '0'}) @@ -554,8 +724,8 @@ class TestContainerController(unittest.TestCase): self.assertEquals(eval(resp.body), []) # fill the container for i in range(3): - req = Request.blank('/sda1/p/a/jsonc/%s'%i, environ= - {'REQUEST_METHOD': 'PUT', + req = Request.blank('/sda1/p/a/jsonc/%s' % i, environ={ + 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', @@ -615,8 +785,8 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 204) # fill the container for i in range(3): - req = Request.blank('/sda1/p/a/plainc/%s'%i, environ= - {'REQUEST_METHOD': 'PUT', + req = Request.blank('/sda1/p/a/plainc/%s' % i, environ={ + 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', @@ -669,13 +839,14 @@ class TestContainerController(unittest.TestCase): def test_GET_json_last_modified(self): # make a container - req = Request.blank('/sda1/p/a/jsonc', environ={'REQUEST_METHOD': 'PUT', + req = Request.blank('/sda1/p/a/jsonc', environ={ + 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = self.controller.PUT(req) for i, d in [(0, 1.5), - (1, 1.0),]: - req = Request.blank('/sda1/p/a/jsonc/%s'%i, environ= - {'REQUEST_METHOD': 'PUT', + (1, 1.0), ]: + req = Request.blank('/sda1/p/a/jsonc/%s' % i, environ={ + 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': d, 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', @@ -693,7 +864,7 @@ class TestContainerController(unittest.TestCase): "hash":"x", "bytes":0, "content_type":"text/plain", - "last_modified":"1970-01-01T00:00:01.000000"},] + "last_modified":"1970-01-01T00:00:01.000000"}, ] req = Request.blank('/sda1/p/a/jsonc?format=json', environ={'REQUEST_METHOD': 'GET'}) @@ -709,8 +880,9 @@ class TestContainerController(unittest.TestCase): resp = self.controller.PUT(req) # fill the container for i in range(3): - req = Request.blank('/sda1/p/a/xmlc/%s'%i, environ= - {'REQUEST_METHOD': 'PUT', + req = Request.blank('/sda1/p/a/xmlc/%s' % i, + environ={ + 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', @@ -772,7 +944,7 @@ class TestContainerController(unittest.TestCase): resp = self.controller.PUT(req) # fill the container for i in range(3): - req = Request.blank('/sda1/p/a/c/%s'%i, environ= {'REQUEST_METHOD': 'PUT', + req = Request.blank('/sda1/p/a/c/%s' % i, environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = self.controller.PUT(req) @@ -781,7 +953,7 @@ class TestContainerController(unittest.TestCase): req = Request.blank('/sda1/p/a/c?limit=2&marker=1', environ={'REQUEST_METHOD': 'GET'}) resp = self.controller.GET(req) result = resp.body.split() - self.assertEquals(result, ['2',]) + self.assertEquals(result, ['2', ]) def test_weird_content_types(self): snowman = u'\u2603' @@ -789,7 +961,8 @@ class TestContainerController(unittest.TestCase): 'HTTP_X_TIMESTAMP': '0'}) resp = self.controller.PUT(req) for i, ctype in enumerate((snowman.encode('utf-8'), 'text/plain; "utf-8"')): - req = Request.blank('/sda1/p/a/c/%s'%i, environ= {'REQUEST_METHOD': 'PUT', + req = Request.blank('/sda1/p/a/c/%s' % i, environ={ + 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': ctype, 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = self.controller.PUT(req) @@ -822,8 +995,9 @@ class TestContainerController(unittest.TestCase): resp = self.controller.PUT(req) # fill the container for i in range(3): - req = Request.blank('/sda1/p/a/c/%s'%i, environ= - {'REQUEST_METHOD': 'PUT', + req = Request.blank('/sda1/p/a/c/%s' % i, + environ={ + 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', @@ -834,15 +1008,16 @@ class TestContainerController(unittest.TestCase): req = Request.blank('/sda1/p/a/c?limit=2', environ={'REQUEST_METHOD': 'GET'}) resp = self.controller.GET(req) result = resp.body.split() - self.assertEquals(result, ['0','1']) + self.assertEquals(result, ['0', '1']) def test_GET_prefix(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = self.controller.PUT(req) for i in ('a1', 'b1', 'a2', 'b2', 'a3', 'b3'): - req = Request.blank('/sda1/p/a/c/%s'%i, environ= - {'REQUEST_METHOD': 'PUT', + req = Request.blank('/sda1/p/a/c/%s' % i, + environ={ + 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', @@ -851,15 +1026,16 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a/c?prefix=a', environ={'REQUEST_METHOD': 'GET'}) resp = self.controller.GET(req) - self.assertEquals(resp.body.split(), ['a1','a2', 'a3']) + self.assertEquals(resp.body.split(), ['a1', 'a2', 'a3']) def test_GET_delimiter(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = self.controller.PUT(req) for i in ('US-TX-A', 'US-TX-B', 'US-OK-A', 'US-OK-B', 'US-UT-A'): - req = Request.blank('/sda1/p/a/c/%s'%i, environ= - {'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', + req = Request.blank('/sda1/p/a/c/%s' % i, + environ={ + 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = self.controller.PUT(req) @@ -868,15 +1044,18 @@ class TestContainerController(unittest.TestCase): environ={'REQUEST_METHOD': 'GET'}) resp = self.controller.GET(req) self.assertEquals(simplejson.loads(resp.body), - [{"subdir":"US-OK-"},{"subdir":"US-TX-"},{"subdir":"US-UT-"}]) + [{"subdir": "US-OK-"}, + {"subdir": "US-TX-"}, + {"subdir": "US-UT-"}]) def test_GET_delimiter_xml(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = self.controller.PUT(req) for i in ('US-TX-A', 'US-TX-B', 'US-OK-A', 'US-OK-B', 'US-UT-A'): - req = Request.blank('/sda1/p/a/c/%s'%i, environ= - {'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', + req = Request.blank('/sda1/p/a/c/%s' % i, + environ={ + 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = self.controller.PUT(req) @@ -894,8 +1073,9 @@ class TestContainerController(unittest.TestCase): 'HTTP_X_TIMESTAMP': '0'}) resp = self.controller.PUT(req) for i in ('US/TX', 'US/TX/B', 'US/OK', 'US/OK/B', 'US/UT/A'): - req = Request.blank('/sda1/p/a/c/%s'%i, environ= - {'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', + req = Request.blank('/sda1/p/a/c/%s' % i, + environ={ + 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = self.controller.PUT(req) @@ -904,17 +1084,27 @@ class TestContainerController(unittest.TestCase): environ={'REQUEST_METHOD': 'GET'}) resp = self.controller.GET(req) self.assertEquals(simplejson.loads(resp.body), - [{"name":"US/OK","hash":"x","bytes":0,"content_type":"text/plain", + [{"name":"US/OK", "hash":"x", "bytes":0, "content_type":"text/plain", "last_modified":"1970-01-01T00:00:01.000000"}, - {"name":"US/TX","hash":"x","bytes":0,"content_type":"text/plain", + {"name":"US/TX", "hash":"x", "bytes":0, "content_type":"text/plain", "last_modified":"1970-01-01T00:00:01.000000"}]) + def test_GET_insufficient_storage(self): + self.controller = container_server.ContainerController( + {'devices': self.testdir}) + req = Request.blank('/sda-null/p/a/c', environ={'REQUEST_METHOD': 'GET', + 'HTTP_X_TIMESTAMP': '1'}) + resp = self.controller.GET(req) + self.assertEquals(resp.status_int, 507) + def test_through_call(self): inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() + def start_response(*args): outbuf.writelines(args) + self.controller.__call__({'REQUEST_METHOD': 'GET', 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/a/c', @@ -937,8 +1127,10 @@ class TestContainerController(unittest.TestCase): inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() + def start_response(*args): outbuf.writelines(args) + self.controller.__call__({'REQUEST_METHOD': 'GET', 'SCRIPT_NAME': '', 'PATH_INFO': '/bob', @@ -957,12 +1149,40 @@ class TestContainerController(unittest.TestCase): self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '400 ') + def test_through_call_invalid_path_utf8(self): + inbuf = StringIO() + errbuf = StringIO() + outbuf = StringIO() + + def start_response(*args): + outbuf.writelines(args) + + self.controller.__call__({'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'PATH_INFO': '\x00', + 'SERVER_NAME': '127.0.0.1', + 'SERVER_PORT': '8080', + 'SERVER_PROTOCOL': 'HTTP/1.0', + 'CONTENT_LENGTH': '0', + 'wsgi.version': (1, 0), + 'wsgi.url_scheme': 'http', + 'wsgi.input': inbuf, + 'wsgi.errors': errbuf, + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.run_once': False}, + start_response) + self.assertEquals(errbuf.getvalue(), '') + self.assertEquals(outbuf.getvalue()[:4], '412 ') + def test_invalid_method_doesnt_exist(self): inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() + def start_response(*args): outbuf.writelines(args) + self.controller.__call__({'REQUEST_METHOD': 'method_doesnt_exist', 'PATH_INFO': '/sda1/p/a/c'}, start_response) @@ -973,8 +1193,10 @@ class TestContainerController(unittest.TestCase): inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() + def start_response(*args): outbuf.writelines(args) + self.controller.__call__({'REQUEST_METHOD': '__init__', 'PATH_INFO': '/sda1/p/a/c'}, start_response) @@ -1082,8 +1304,10 @@ class TestContainerController(unittest.TestCase): def test_updating_multiple_container_servers(self): http_connect_args = [] + def fake_http_connect(ipaddr, port, device, partition, method, path, headers=None, query_string=None, ssl=False): + class SuccessfulFakeConn(object): @property def status(self): @@ -1101,7 +1325,7 @@ class TestContainerController(unittest.TestCase): 'headers': headers, 'query_string': query_string} http_connect_args.append( - dict((k,v) for k,v in captured_args.iteritems() + dict((k, v) for k, v in captured_args.iteritems() if v is not None)) req = Request.blank( @@ -1154,4 +1378,3 @@ class TestContainerController(unittest.TestCase): if __name__ == '__main__': unittest.main() - diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 097b17b02c..3e9640fb57 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -16,20 +16,16 @@ from __future__ import with_statement import cPickle as pickle import logging -from logging.handlers import SysLogHandler import os import sys import unittest import urlparse import signal -from ConfigParser import ConfigParser from contextlib import contextmanager -from cStringIO import StringIO from gzip import GzipFile -from httplib import HTTPException from shutil import rmtree import time -from urllib import unquote, quote +from urllib import quote from hashlib import md5 from tempfile import mkdtemp import random @@ -38,7 +34,7 @@ import eventlet from eventlet import sleep, spawn, Timeout, util, wsgi, listen import simplejson -from test.unit import connect_tcp, readuntil2crlfs, FakeLogger +from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect from swift.proxy import server as proxy_server from swift.account import server as account_server from swift.container import server as container_server @@ -196,130 +192,6 @@ def sortHeaderNames(headerNames): return ', '.join(headers) -def fake_http_connect(*code_iter, **kwargs): - - class FakeConn(object): - - def __init__(self, status, etag=None, body='', timestamp='1', - expect_status=None): - self.status = status - if expect_status is None: - self.expect_status = self.status - else: - self.expect_status = expect_status - self.reason = 'Fake' - self.host = '1.2.3.4' - self.port = '1234' - self.sent = 0 - self.received = 0 - self.etag = etag - self.body = body - self.timestamp = timestamp - - def getresponse(self): - if kwargs.get('raise_exc'): - raise Exception('test') - if kwargs.get('raise_timeout_exc'): - raise Timeout() - return self - - def getexpect(self): - if self.expect_status == -2: - raise HTTPException() - if self.expect_status == -3: - return FakeConn(507) - if self.expect_status == -4: - return FakeConn(201) - return FakeConn(100) - - def getheaders(self): - etag = self.etag - if not etag: - if isinstance(self.body, str): - etag = '"' + md5(self.body).hexdigest() + '"' - else: - etag = '"68b329da9893e34099c7d8ad5cb9c940"' - - headers = {'content-length': len(self.body), - 'content-type': 'x-application/test', - 'x-timestamp': self.timestamp, - 'last-modified': self.timestamp, - 'x-object-meta-test': 'testing', - 'etag': etag, - 'x-works': 'yes', - 'x-account-container-count': kwargs.get('count', 12345)} - if not self.timestamp: - del headers['x-timestamp'] - try: - if container_ts_iter.next() is False: - headers['x-container-timestamp'] = '1' - except StopIteration: - pass - if 'slow' in kwargs: - headers['content-length'] = '4' - if 'headers' in kwargs: - headers.update(kwargs['headers']) - return headers.items() - - def read(self, amt=None): - if 'slow' in kwargs: - if self.sent < 4: - self.sent += 1 - sleep(0.1) - return ' ' - rv = self.body[:amt] - self.body = self.body[amt:] - return rv - - def send(self, amt=None): - if 'slow' in kwargs: - if self.received < 4: - self.received += 1 - sleep(0.1) - - def getheader(self, name, default=None): - return dict(self.getheaders()).get(name.lower(), default) - - timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) - etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) - x = kwargs.get('missing_container', [False] * len(code_iter)) - if not isinstance(x, (tuple, list)): - x = [x] * len(code_iter) - container_ts_iter = iter(x) - code_iter = iter(code_iter) - static_body = kwargs.get('body', None) - body_iter = kwargs.get('body_iter', None) - if body_iter: - body_iter = iter(body_iter) - - def connect(*args, **ckwargs): - if 'give_content_type' in kwargs: - if len(args) >= 7 and 'Content-Type' in args[6]: - kwargs['give_content_type'](args[6]['Content-Type']) - else: - kwargs['give_content_type']('') - if 'give_connect' in kwargs: - kwargs['give_connect'](*args, **ckwargs) - status = code_iter.next() - if isinstance(status, tuple): - status, expect_status = status - else: - expect_status = status - etag = etag_iter.next() - timestamp = timestamps_iter.next() - - if status <= 0: - raise HTTPException() - if body_iter is None: - body = static_body or '' - else: - body = body_iter.next() - return FakeConn(status, etag, body=body, timestamp=timestamp, - expect_status=expect_status) - - return connect - - class FakeRing(object): def __init__(self, replicas=3): @@ -883,7 +755,7 @@ class TestObjectController(unittest.TestCase): 'X-Storage-Token: t\r\n' 'Content-Length: %s\r\n' 'Content-Type: application/octet-stream\r\n' - '\r\n%s' % (path, str(len(obj)), obj)) + '\r\n%s' % (path, str(len(obj)), obj)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' @@ -2435,7 +2307,7 @@ class TestObjectController(unittest.TestCase): headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) self.app.update_request(req) - + class LargeResponseBody(object): def __len__(self): @@ -2451,7 +2323,6 @@ class TestObjectController(unittest.TestCase): resp = controller.PUT(req) self.assertEquals(resp.status_int, 413) - def test_COPY(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') @@ -2581,7 +2452,6 @@ class TestObjectController(unittest.TestCase): resp = controller.COPY(req) self.assertEquals(resp.status_int, 413) - def test_COPY_newest(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') @@ -3986,7 +3856,7 @@ class TestObjectController(unittest.TestCase): 'X-Auth-Token: t\r\n' 'Content-Length: %s\r\n' 'Content-Type: application/octet-stream\r\n' - '\r\n%s' % (obj_len, 'a' * obj_len)) + '\r\n%s' % (obj_len, 'a' * obj_len)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' @@ -4175,6 +4045,7 @@ class TestObjectController(unittest.TestCase): 'X-Container-Host', 'X-Container-Device']) seen_headers = [] + def capture_headers(ipaddr, port, device, partition, method, path, headers=None, query_string=None): captured = {} @@ -4188,7 +4059,7 @@ class TestObjectController(unittest.TestCase): set_http_connect(*connect_args, give_connect=capture_headers, **kwargs) resp = controller_call(req) - self.assertEqual(2, resp.status_int // 100) # sanity check + self.assertEqual(2, resp.status_int // 100) # sanity check # don't care about the account/container HEADs, so chuck # the first two requests @@ -4323,7 +4194,6 @@ class TestObjectController(unittest.TestCase): 'X-Delete-At-Partition': None, 'X-Delete-At-Device': None}]) - def test_PUT_x_delete_at_with_more_container_replicas(self): self.app.container_ring.set_replicas(4) self.app.expiring_objects_account = 'expires' @@ -5017,7 +4887,7 @@ class TestContainerController(unittest.TestCase): set_http_connect(*connect_args, give_connect=capture_headers, **kwargs) resp = controller_call(req) - self.assertEqual(2, resp.status_int // 100) # sanity check + self.assertEqual(2, resp.status_int // 100) # sanity check # don't care about the account HEAD, so throw away the # first element @@ -5140,7 +5010,7 @@ class TestAccountController(unittest.TestCase): self.app.allow_account_management = False controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/account', {'REQUEST_METHOD': 'OPTIONS'}, - headers = {'Origin': 'http://foo.com', + headers={'Origin': 'http://foo.com', 'Access-Control-Request-Method': 'GET'}) req.content_length = 0 resp = controller.OPTIONS(req)