openstack
/
swift
Code Issues Proposed changes
OpenStack Storage (Swift)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1814 Commits
20 Branches
6 Tags
164 MiB
 
 
Tree: 40cbff9fd3
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '40cbff9fd3'
${ noResults }
swift/test/unit/proxy/test_server.py

4429 lines
198 KiB
Raw Blame History

# Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import with_statement
import cPickle as pickle
import logging
from logging.handlers import SysLogHandler
import os
import sys
import unittest
from nose import SkipTest
from ConfigParser import ConfigParser
from contextlib import contextmanager
from cStringIO import StringIO
from gzip import GzipFile
from httplib import HTTPException
from shutil import rmtree
from time import time
from urllib import unquote, quote
from hashlib import md5
from tempfile import mkdtemp
import eventlet
from eventlet import sleep, spawn, Timeout, util, wsgi, listen
import simplejson
from webob import Request, Response
from webob.exc import HTTPNotFound, HTTPUnauthorized
from test.unit import connect_tcp, readuntil2crlfs
from swift.proxy import server as proxy_server
from swift.account import server as account_server
from swift.container import server as container_server
from swift.obj import server as object_server
from swift.common import ring
from swift.common.constraints import MAX_META_NAME_LENGTH, \
MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, MAX_FILE_SIZE
from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
from swift.common.wsgi import monkey_patch_mimetools
# mocks
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
def setup():
global _testdir, _test_servers, _test_sockets, \
_orig_container_listing_limit, _test_coros
monkey_patch_mimetools()
# Since we're starting up a lot here, we're going to test more than
# just chunked puts; we're also going to test parts of
# proxy_server.Application we couldn't get to easily otherwise.
_testdir = \
os.path.join(mkdtemp(), 'tmp_test_proxy_server_chunked')
mkdirs(_testdir)
rmtree(_testdir)
mkdirs(os.path.join(_testdir, 'sda1'))
mkdirs(os.path.join(_testdir, 'sda1', 'tmp'))
mkdirs(os.path.join(_testdir, 'sdb1'))
mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))
_orig_container_listing_limit = proxy_server.CONTAINER_LISTING_LIMIT
conf = {'devices': _testdir, 'swift_dir': _testdir,
'mount_check': 'false', 'allowed_headers':
'content-encoding, x-object-manifest, content-disposition, foo',
'allow_versions': 'True'}
prolis = listen(('localhost', 0))
acc1lis = listen(('localhost', 0))
acc2lis = listen(('localhost', 0))
con1lis = listen(('localhost', 0))
con2lis = listen(('localhost', 0))
obj1lis = listen(('localhost', 0))
obj2lis = listen(('localhost', 0))
_test_sockets = \
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis)
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': acc1lis.getsockname()[1]},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': acc2lis.getsockname()[1]}], 30),
GzipFile(os.path.join(_testdir, 'account.ring.gz'), 'wb'))
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': con1lis.getsockname()[1]},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': con2lis.getsockname()[1]}], 30),
GzipFile(os.path.join(_testdir, 'container.ring.gz'), 'wb'))
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': obj1lis.getsockname()[1]},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': obj2lis.getsockname()[1]}], 30),
GzipFile(os.path.join(_testdir, 'object.ring.gz'), 'wb'))
prosrv = proxy_server.Application(conf, FakeMemcacheReturnsNone())
acc1srv = account_server.AccountController(conf)
acc2srv = account_server.AccountController(conf)
con1srv = container_server.ContainerController(conf)
con2srv = container_server.ContainerController(conf)
obj1srv = object_server.ObjectController(conf)
obj2srv = object_server.ObjectController(conf)
_test_servers = \
(prosrv, acc1srv, acc2srv, con2srv, con2srv, obj1srv, obj2srv)
nl = NullLogger()
prospa = spawn(wsgi.server, prolis, prosrv, nl)
acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl)
acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl)
con1spa = spawn(wsgi.server, con1lis, con1srv, nl)
con2spa = spawn(wsgi.server, con2lis, con2srv, nl)
obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl)
obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl)
_test_coros = \
(prospa, acc1spa, acc2spa, con2spa, con2spa, obj1spa, obj2spa)
# Create account
ts = normalize_timestamp(time())
partition, nodes = prosrv.account_ring.get_nodes('a')
for node in nodes:
conn = proxy_server.http_connect(node['ip'], node['port'],
node['device'], partition, 'PUT', '/a',
{'X-Timestamp': ts, 'x-trans-id': 'test'})
resp = conn.getresponse()
assert(resp.status == 201)
# Create container
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nX-Auth-Token: t\r\n'
'Content-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
assert(headers[:len(exp)] == exp)
def teardown():
for server in _test_coros:
server.kill()
proxy_server.CONTAINER_LISTING_LIMIT = _orig_container_listing_limit
rmtree(os.path.dirname(_testdir))
def fake_http_connect(*code_iter, **kwargs):
class FakeConn(object):
def __init__(self, status, etag=None, body='', timestamp='1'):
self.status = status
self.reason = 'Fake'
self.host = '1.2.3.4'
self.port = '1234'
self.sent = 0
self.received = 0
self.etag = etag
self.body = body
self.timestamp = timestamp
def getresponse(self):
if kwargs.get('raise_exc'):
raise Exception('test')
if kwargs.get('raise_timeout_exc'):
raise Timeout()
return self
def getexpect(self):
if self.status == -2:
raise HTTPException()
if self.status == -3:
return FakeConn(507)
return FakeConn(100)
def getheaders(self):
headers = {'content-length': len(self.body),
'content-type': 'x-application/test',
'x-timestamp': self.timestamp,
'last-modified': self.timestamp,
'x-object-meta-test': 'testing',
'etag':
self.etag or '"68b329da9893e34099c7d8ad5cb9c940"',
'x-works': 'yes',
'x-account-container-count': 12345}
if not self.timestamp:
del headers['x-timestamp']
try:
if container_ts_iter.next() is False:
headers['x-container-timestamp'] = '1'
except StopIteration:
pass
if 'slow' in kwargs:
headers['content-length'] = '4'
if 'headers' in kwargs:
headers.update(kwargs['headers'])
return headers.items()
def read(self, amt=None):
if 'slow' in kwargs:
if self.sent < 4:
self.sent += 1
sleep(0.1)
return ' '
rv = self.body[:amt]
self.body = self.body[amt:]
return rv
def send(self, amt=None):
if 'slow' in kwargs:
if self.received < 4:
self.received += 1
sleep(0.1)
def getheader(self, name, default=None):
return dict(self.getheaders()).get(name.lower(), default)
timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter))
etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter))
x = kwargs.get('missing_container', [False] * len(code_iter))
if not isinstance(x, (tuple, list)):
x = [x] * len(code_iter)
container_ts_iter = iter(x)
code_iter = iter(code_iter)
def connect(*args, **ckwargs):
if 'give_content_type' in kwargs:
if len(args) >= 7 and 'content_type' in args[6]:
kwargs['give_content_type'](args[6]['content-type'])
else:
kwargs['give_content_type']('')
if 'give_connect' in kwargs:
kwargs['give_connect'](*args, **ckwargs)
status = code_iter.next()
etag = etag_iter.next()
timestamp = timestamps_iter.next()
if status <= 0:
raise HTTPException()
return FakeConn(status, etag, body=kwargs.get('body', ''),
timestamp=timestamp)
return connect
class FakeRing(object):
def __init__(self):
# 9 total nodes (6 more past the initial 3) is the cap, no matter if
# this is set higher.
self.max_more_nodes = 0
self.devs = {}
self.replica_count = 3
def get_nodes(self, account, container=None, obj=None):
devs = []
for x in xrange(3):
devs.append(self.devs.get(x))
if devs[x] is None:
self.devs[x] = devs[x] = \
{'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sda'}
return 1, devs
def get_part_nodes(self, part):
return self.get_nodes('blah')[1]
def get_more_nodes(self, nodes):
# 9 is the true cap
for x in xrange(3, min(3 + self.max_more_nodes, 9)):
yield {'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sda'}
class FakeMemcache(object):
def __init__(self):
self.store = {}
def get(self, key):
return self.store.get(key)
def keys(self):
return self.store.keys()
def set(self, key, value, timeout=0):
self.store[key] = value
return True
def incr(self, key, timeout=0):
self.store[key] = self.store.setdefault(key, 0) + 1
return self.store[key]
@contextmanager
def soft_lock(self, key, timeout=0, retries=5):
yield True
def delete(self, key):
try:
del self.store[key]
except Exception:
pass
return True
class FakeMemcacheReturnsNone(FakeMemcache):
def get(self, key):
# Returns None as the timestamp of the container; assumes we're only
# using the FakeMemcache for container existence checks.
return None
class NullLoggingHandler(logging.Handler):
def emit(self, record):
pass
@contextmanager
def save_globals():
orig_http_connect = getattr(proxy_server, 'http_connect', None)
orig_account_info = getattr(proxy_server.Controller, 'account_info', None)
try:
yield True
finally:
proxy_server.http_connect = orig_http_connect
proxy_server.Controller.account_info = orig_account_info
# tests
class TestController(unittest.TestCase):
def setUp(self):
self.account_ring = FakeRing()
self.container_ring = FakeRing()
self.memcache = FakeMemcache()
app = proxy_server.Application(None, self.memcache,
account_ring=self.account_ring,
container_ring=self.container_ring,
object_ring=FakeRing())
self.controller = proxy_server.Controller(app)
self.account = 'some_account'
self.container = 'some_container'
self.read_acl = 'read_acl'
self.write_acl = 'write_acl'
def check_account_info_return(self, partition, nodes, is_none=False):
if is_none:
p, n = None, None
else:
p, n = self.account_ring.get_nodes(self.account)
self.assertEqual(p, partition)
self.assertEqual(n, nodes)
def test_make_requests(self):
with save_globals():
proxy_server.http_connect = fake_http_connect(200)
partition, nodes, count = \
self.controller.account_info(self.account)
proxy_server.http_connect = fake_http_connect(201,
raise_timeout_exc=True)
self.controller._make_request(nodes, partition, 'POST',
'/', '', '')
# tests if 200 is cached and used
def test_account_info_200(self):
with save_globals():
proxy_server.http_connect = fake_http_connect(200)
partition, nodes, count = \
self.controller.account_info(self.account)
self.check_account_info_return(partition, nodes)
self.assertEquals(count, 12345)
cache_key = proxy_server.get_account_memcache_key(self.account)
self.assertEquals({'status': 200, 'container_count': 12345},
self.memcache.get(cache_key))
proxy_server.http_connect = fake_http_connect()
partition, nodes, count = \
self.controller.account_info(self.account)
self.check_account_info_return(partition, nodes)
self.assertEquals(count, 12345)
# tests if 404 is cached and used
def test_account_info_404(self):
with save_globals():
proxy_server.http_connect = fake_http_connect(404, 404, 404)
partition, nodes, count = \
self.controller.account_info(self.account)
self.check_account_info_return(partition, nodes, True)
self.assertEquals(count, None)
cache_key = proxy_server.get_account_memcache_key(self.account)
self.assertEquals({'status': 404, 'container_count': 0},
self.memcache.get(cache_key))
proxy_server.http_connect = fake_http_connect()
partition, nodes, count = \
self.controller.account_info(self.account)
self.check_account_info_return(partition, nodes, True)
self.assertEquals(count, None)
# tests if some http status codes are not cached
def test_account_info_no_cache(self):
def test(*status_list):
proxy_server.http_connect = fake_http_connect(*status_list)
partition, nodes, count = \
self.controller.account_info(self.account)
self.assertEqual(len(self.memcache.keys()), 0)
self.check_account_info_return(partition, nodes, True)
self.assertEquals(count, None)
with save_globals():
test(503, 404, 404)
test(404, 404, 503)
test(404, 507, 503)
test(503, 503, 503)
def test_account_info_account_autocreate(self):
with save_globals():
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 201, 201, 201)
partition, nodes, count = \
self.controller.account_info(self.account, autocreate=False)
self.check_account_info_return(partition, nodes, is_none=True)
self.assertEquals(count, None)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 201, 201, 201)
partition, nodes, count = \
self.controller.account_info(self.account)
self.check_account_info_return(partition, nodes, is_none=True)
self.assertEquals(count, None)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 201, 201, 201)
partition, nodes, count = \
self.controller.account_info(self.account, autocreate=True)
self.check_account_info_return(partition, nodes)
self.assertEquals(count, 0)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 503, 201, 201)
partition, nodes, count = \
self.controller.account_info(self.account, autocreate=True)
self.check_account_info_return(partition, nodes)
self.assertEquals(count, 0)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 503, 201, 503)
exc = None
try:
partition, nodes, count = \
self.controller.account_info(self.account, autocreate=True)
except Exception, err:
exc = err
self.assertEquals(str(exc),
"Could not autocreate account '/some_account'")
def check_container_info_return(self, ret, is_none=False):
if is_none:
partition, nodes, read_acl, write_acl = None, None, None, None
else:
partition, nodes = self.container_ring.get_nodes(self.account,
self.container)
read_acl, write_acl = self.read_acl, self.write_acl
self.assertEqual(partition, ret[0])
self.assertEqual(nodes, ret[1])
self.assertEqual(read_acl, ret[2])
self.assertEqual(write_acl, ret[3])
def test_container_info_invalid_account(self):
def account_info(self, account, autocreate=False):
return None, None
with save_globals():
proxy_server.Controller.account_info = account_info
ret = self.controller.container_info(self.account,
self.container)
self.check_container_info_return(ret, True)
# tests if 200 is cached and used
def test_container_info_200(self):
def account_info(self, account, autocreate=False):
return True, True, 0
with save_globals():
headers = {'x-container-read': self.read_acl,
'x-container-write': self.write_acl}
proxy_server.Controller.account_info = account_info
proxy_server.http_connect = fake_http_connect(200,
headers=headers)
ret = self.controller.container_info(self.account,
self.container)
self.check_container_info_return(ret)
cache_key = proxy_server.get_container_memcache_key(self.account,
self.container)
cache_value = self.memcache.get(cache_key)
self.assertTrue(isinstance(cache_value, dict))
self.assertEquals(200, cache_value.get('status'))
proxy_server.http_connect = fake_http_connect()
ret = self.controller.container_info(self.account,
self.container)
self.check_container_info_return(ret)
# tests if 404 is cached and used
def test_container_info_404(self):
def account_info(self, account, autocreate=False):
return True, True, 0
with save_globals():
proxy_server.Controller.account_info = account_info
proxy_server.http_connect = fake_http_connect(404, 404, 404)
ret = self.controller.container_info(self.account,
self.container)
self.check_container_info_return(ret, True)
cache_key = proxy_server.get_container_memcache_key(self.account,
self.container)
cache_value = self.memcache.get(cache_key)
self.assertTrue(isinstance(cache_value, dict))
self.assertEquals(404, cache_value.get('status'))
proxy_server.http_connect = fake_http_connect()
ret = self.controller.container_info(self.account,
self.container)
self.check_container_info_return(ret, True)
# tests if some http status codes are not cached
def test_container_info_no_cache(self):
def test(*status_list):
proxy_server.http_connect = fake_http_connect(*status_list)
ret = self.controller.container_info(self.account,
self.container)
self.assertEqual(len(self.memcache.keys()), 0)
self.check_container_info_return(ret, True)
with save_globals():
test(503, 404, 404)
test(404, 404, 503)
test(404, 507, 503)
test(503, 503, 503)
class TestProxyServer(unittest.TestCase):
def test_access_log(self):
class MyApp(proxy_server.Application):
def handle_request(self, req):
resp = Response(request=req)
req.response = resp
req.start_time = time()
return resp
def start_response(*args):
pass
class MockLogger():
def __init__(self):
self.buffer = StringIO()
def info(self, msg, args=None):
if args:
msg = msg % args
self.buffer.write(msg)
def strip_value(self):
rv = self.buffer.getvalue()
self.buffer.truncate(0)
return rv
class SnarfStream(object):
# i can't seem to subclass cStringIO
def __init__(self, *args, **kwargs):
self.sio = StringIO()
def strip_value(self):
rv = self.getvalue().strip()
self.truncate(0)
return rv
def __getattr__(self, name):
try:
return object.__getattr__(self, name)
except AttributeError:
try:
return getattr(self.sio, name)
except AttributeError:
return self.__getattribute__(name)
snarf = SnarfStream()
_orig_get_logger = proxy_server.get_logger
def mock_get_logger(*args, **kwargs):
if kwargs.get('log_route') != 'proxy-access':
return _orig_get_logger(*args, **kwargs)
kwargs['log_route'] = 'snarf'
logger = _orig_get_logger(*args, **kwargs)
if [h for h in logger.logger.handlers if
isinstance(h, logging.StreamHandler) and h.stream is snarf]:
# snarf handler already setup!
return logger
formatter = logger.logger.handlers[0].formatter
formatter._fmt += ' %(levelname)s'
snarf_handler = logging.StreamHandler(snarf)
snarf_handler.setFormatter(formatter)
logger.logger.addHandler(snarf_handler)
return logger
def test_conf(conf):
app = MyApp(conf, memcache=FakeMemcache(), account_ring=FakeRing(),
container_ring=FakeRing(), object_ring=FakeRing())
req = Request.blank('')
app(req.environ, start_response)
try:
proxy_server.get_logger = mock_get_logger
test_conf({})
line = snarf.strip_value()
self.assert_(line.startswith('swift'))
self.assert_(line.endswith('INFO'))
test_conf({'log_name': 'snarf-test'})
line = snarf.strip_value()
self.assert_(line.startswith('snarf-test'))
self.assert_(line.endswith('INFO'))
test_conf({'log_name': 'snarf-test', 'log_level': 'ERROR'})
line = snarf.strip_value()
self.assertFalse(line)
test_conf({'log_name': 'snarf-test', 'log_level': 'ERROR',
'access_log_name': 'access-test',
'access_log_level': 'INFO'})
line = snarf.strip_value()
self.assert_(line.startswith('access-test'))
self.assert_(line.endswith('INFO'))
# test facility
def get_facility(logger):
h = [h for h in logger.logger.handlers if
isinstance(h, SysLogHandler)][0]
return h.facility
conf = {'log_facility': 'LOG_LOCAL0'}
app = MyApp(conf, memcache=FakeMemcache(), account_ring=FakeRing(),
container_ring=FakeRing(), object_ring=FakeRing())
self.assertEquals(get_facility(app.logger),
SysLogHandler.LOG_LOCAL0)
self.assertEquals(get_facility(app.access_logger),
SysLogHandler.LOG_LOCAL0)
conf = {'log_facility': 'LOG_LOCAL0',
'access_log_facility': 'LOG_LOCAL1'}
app = MyApp(conf, memcache=FakeMemcache(), account_ring=FakeRing(),
container_ring=FakeRing(), object_ring=FakeRing())
self.assertEquals(get_facility(app.logger),
SysLogHandler.LOG_LOCAL0)
self.assertEquals(get_facility(app.access_logger),
SysLogHandler.LOG_LOCAL1)
conf = {'access_log_facility': 'LOG_LOCAL1'}
app = MyApp(conf, memcache=FakeMemcache(), account_ring=FakeRing(),
container_ring=FakeRing(), object_ring=FakeRing())
self.assertEquals(get_facility(app.logger),
SysLogHandler.LOG_LOCAL0)
self.assertEquals(get_facility(app.access_logger),
SysLogHandler.LOG_LOCAL1)
finally:
proxy_server.get_logger = _orig_get_logger
def test_unhandled_exception(self):
class MyApp(proxy_server.Application):
def get_controller(self, path):
raise Exception('this shouldnt be caught')
app = MyApp(None, FakeMemcache(), account_ring=FakeRing(),
container_ring=FakeRing(), object_ring=FakeRing())
req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'})
app.update_request(req)
resp = app.handle_request(req)
self.assertEquals(resp.status_int, 500)
def test_calls_authorize_allow(self):
called = [False]
def authorize(req):
called[0] = True
with save_globals():
proxy_server.http_connect = fake_http_connect(200)
app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(), container_ring=FakeRing(),
object_ring=FakeRing())
req = Request.blank('/v1/a')
req.environ['swift.authorize'] = authorize
app.update_request(req)
resp = app.handle_request(req)
self.assert_(called[0])
def test_calls_authorize_deny(self):
called = [False]
def authorize(req):
called[0] = True
return HTTPUnauthorized(request=req)
app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(), container_ring=FakeRing(),
object_ring=FakeRing())
req = Request.blank('/v1/a')
req.environ['swift.authorize'] = authorize
app.update_request(req)
resp = app.handle_request(req)
self.assert_(called[0])
def test_negative_content_length(self):
swift_dir = mkdtemp()
try:
baseapp = proxy_server.BaseApplication({'swift_dir': swift_dir},
FakeMemcache(), NullLoggingHandler(), FakeRing(), FakeRing(),
FakeRing())
resp = baseapp.handle_request(
Request.blank('/', environ={'CONTENT_LENGTH': '-1'}))
self.assertEquals(resp.status, '400 Bad Request')
self.assertEquals(resp.body, 'Invalid Content-Length')
resp = baseapp.handle_request(
Request.blank('/', environ={'CONTENT_LENGTH': '-123'}))
self.assertEquals(resp.status, '400 Bad Request')
self.assertEquals(resp.body, 'Invalid Content-Length')
finally:
rmtree(swift_dir, ignore_errors=True)
def test_denied_host_header(self):
swift_dir = mkdtemp()
try:
baseapp = proxy_server.BaseApplication({'swift_dir': swift_dir,
'deny_host_headers': 'invalid_host.com'},
FakeMemcache(), NullLoggingHandler(), FakeRing(), FakeRing(),
FakeRing())
resp = baseapp.handle_request(
Request.blank('/v1/a/c/o',
environ={'HTTP_HOST': 'invalid_host.com'}))
self.assertEquals(resp.status, '403 Forbidden')
finally:
rmtree(swift_dir, ignore_errors=True)
class TestObjectController(unittest.TestCase):
def setUp(self):
self.app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(), container_ring=FakeRing(),
object_ring=FakeRing())
monkey_patch_mimetools()
def tearDown(self):
proxy_server.CONTAINER_LISTING_LIMIT = _orig_container_listing_limit
def assert_status_map(self, method, statuses, expected, raise_exc=False):
with save_globals():
kwargs = {}
if raise_exc:
kwargs['raise_exc'] = raise_exc
proxy_server.http_connect = fake_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', headers={'Content-Length': '0',
'Content-Type': 'text/plain'})
self.app.update_request(req)
res = method(req)
self.assertEquals(res.status_int, expected)
# repeat test
proxy_server.http_connect = fake_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', headers={'Content-Length': '0',
'Content-Type': 'text/plain'})
self.app.update_request(req)
res = method(req)
self.assertEquals(res.status_int, expected)
def test_PUT_auto_content_type(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_content_type(filename, expected):
proxy_server.http_connect = fake_http_connect(201, 201, 201,
give_content_type=lambda content_type:
self.assertEquals(content_type, expected.next()))
req = Request.blank('/a/c/%s' % filename, {})
self.app.update_request(req)
res = controller.PUT(req)
test_content_type('test.jpg', iter(['', '', '', 'image/jpeg',
'image/jpeg', 'image/jpeg']))
test_content_type('test.html', iter(['', '', '', 'text/html',
'text/html', 'text/html']))
test_content_type('test.css', iter(['', '', '', 'text/css',
'text/css', 'text/css']))
def test_custom_mime_types_files(self):
swift_dir = mkdtemp()
try:
with open(os.path.join(swift_dir, 'mime.types'), 'w') as fp:
fp.write('foo/bar foo\n')
ba = proxy_server.BaseApplication({'swift_dir': swift_dir},
FakeMemcache(), NullLoggingHandler(), FakeRing(), FakeRing(),
FakeRing())
self.assertEquals(proxy_server.mimetypes.guess_type('blah.foo')[0],
'foo/bar')
self.assertEquals(proxy_server.mimetypes.guess_type('blah.jpg')[0],
'image/jpeg')
finally:
rmtree(swift_dir, ignore_errors=True)
def test_PUT(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected):
proxy_server.http_connect = fake_http_connect(*statuses)
req = Request.blank('/a/c/o.jpg', {})
req.content_length = 0
self.app.update_request(req)
self.app.memcache.store = {}
res = controller.PUT(req)
expected = str(expected)
self.assertEquals(res.status[:len(expected)], expected)
test_status_map((200, 200, 201, 201, 201), 201)
test_status_map((200, 200, 201, 201, 500), 201)
test_status_map((200, 200, 204, 404, 404), 404)
test_status_map((200, 200, 204, 500, 404), 503)
def test_PUT_connect_exceptions(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected):
proxy_server.http_connect = fake_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/a/c/o.jpg', {})
req.content_length = 0
self.app.update_request(req)
res = controller.PUT(req)
expected = str(expected)
self.assertEquals(res.status[:len(expected)], expected)
test_status_map((200, 200, 201, 201, -1), 201)
test_status_map((200, 200, 201, 201, -2), 201) # expect timeout
test_status_map((200, 200, 201, 201, -3), 201) # error limited
test_status_map((200, 200, 201, -1, -1), 503)
test_status_map((200, 200, 503, 503, -1), 503)
def test_PUT_send_exceptions(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected):
self.app.memcache.store = {}
proxy_server.http_connect = fake_http_connect(*statuses)
req = Request.blank('/a/c/o.jpg',
environ={'REQUEST_METHOD': 'PUT'}, body='some data')
self.app.update_request(req)
res = controller.PUT(req)
expected = str(expected)
self.assertEquals(res.status[:len(expected)], expected)
test_status_map((200, 200, 201, -1, 201), 201)
test_status_map((200, 200, 201, -1, -1), 503)
test_status_map((200, 200, 503, 503, -1), 503)
def test_PUT_max_size(self):
with save_globals():
proxy_server.http_connect = fake_http_connect(201, 201, 201)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
req = Request.blank('/a/c/o', {}, headers={
'Content-Length': str(MAX_FILE_SIZE + 1),
'Content-Type': 'foo/bar'})
self.app.update_request(req)
res = controller.PUT(req)
self.assertEquals(res.status_int, 413)
def test_PUT_getresponse_exceptions(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected):
self.app.memcache.store = {}
proxy_server.http_connect = fake_http_connect(*statuses)
req = Request.blank('/a/c/o.jpg', {})
req.content_length = 0
self.app.update_request(req)
res = controller.PUT(req)
expected = str(expected)
self.assertEquals(res.status[:len(str(expected))],
str(expected))
test_status_map((200, 200, 201, 201, -1), 201)
test_status_map((200, 200, 201, -1, -1), 503)
test_status_map((200, 200, 503, 503, -1), 503)
def test_POST(self):
with save_globals():
self.app.object_post_as_copy = False
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected):
proxy_server.http_connect = fake_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar'})
self.app.update_request(req)
res = controller.POST(req)
expected = str(expected)
self.assertEquals(res.status[:len(expected)], expected)
test_status_map((200, 200, 202, 202, 202), 202)
test_status_map((200, 200, 202, 202, 500), 202)
test_status_map((200, 200, 202, 500, 500), 503)
test_status_map((200, 200, 202, 404, 500), 503)
test_status_map((200, 200, 202, 404, 404), 404)
test_status_map((200, 200, 404, 500, 500), 503)
test_status_map((200, 200, 404, 404, 404), 404)
def test_POST_as_copy(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected):
proxy_server.http_connect = fake_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar'})
self.app.update_request(req)
res = controller.POST(req)
expected = str(expected)
self.assertEquals(res.status[:len(expected)], expected)
test_status_map((200, 200, 200, 200, 200, 202, 202, 202), 202)
test_status_map((200, 200, 200, 200, 200, 202, 202, 500), 202)
test_status_map((200, 200, 200, 200, 200, 202, 500, 500), 503)
test_status_map((200, 200, 200, 200, 200, 202, 404, 500), 503)
test_status_map((200, 200, 200, 200, 200, 202, 404, 404), 404)
test_status_map((200, 200, 200, 200, 200, 404, 500, 500), 503)
test_status_map((200, 200, 200, 200, 200, 404, 404, 404), 404)
def test_DELETE(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected):
proxy_server.http_connect = fake_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {})
self.app.update_request(req)
res = controller.DELETE(req)
self.assertEquals(res.status[:len(str(expected))],
str(expected))
test_status_map((200, 200, 204, 204, 204), 204)
test_status_map((200, 200, 204, 204, 500), 204)
test_status_map((200, 200, 204, 404, 404), 404)
test_status_map((200, 200, 204, 500, 404), 503)
test_status_map((200, 200, 404, 404, 404), 404)
test_status_map((200, 200, 404, 404, 500), 404)
def test_HEAD(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected):
proxy_server.http_connect = fake_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {})
self.app.update_request(req)
res = controller.HEAD(req)
self.assertEquals(res.status[:len(str(expected))],
str(expected))
if expected < 400:
self.assert_('x-works' in res.headers)
self.assertEquals(res.headers['x-works'], 'yes')
self.assert_('accept-ranges' in res.headers)
self.assertEquals(res.headers['accept-ranges'], 'bytes')
test_status_map((200, 200, 200, 404, 404), 200)
test_status_map((200, 200, 200, 500, 404), 200)
test_status_map((200, 200, 304, 500, 404), 304)
test_status_map((200, 200, 404, 404, 404), 404)
test_status_map((200, 200, 404, 404, 500), 404)
test_status_map((200, 200, 500, 500, 500), 503)
def test_HEAD_newest(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected, timestamps,
expected_timestamp):
proxy_server.http_connect = \
fake_http_connect(*statuses, timestamps=timestamps)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'})
self.app.update_request(req)
res = controller.HEAD(req)
self.assertEquals(res.status[:len(str(expected))],
str(expected))
self.assertEquals(res.headers.get('last-modified'),
expected_timestamp)
# acct cont obj obj obj
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '2', '3'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '2'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '1'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3', '3', '1'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, None, None), None)
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, None, '1'), '1')
def test_GET_newest(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected, timestamps,
expected_timestamp):
proxy_server.http_connect = \
fake_http_connect(*statuses, timestamps=timestamps)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'})
self.app.update_request(req)
res = controller.GET(req)
self.assertEquals(res.status[:len(str(expected))],
str(expected))
self.assertEquals(res.headers.get('last-modified'),
expected_timestamp)
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '2', '3'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '2'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '1'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3', '3', '1'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, None, None), None)
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, None, '1'), '1')
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
def test_status_map(statuses, expected, timestamps,
expected_timestamp):
proxy_server.http_connect = \
fake_http_connect(*statuses, timestamps=timestamps)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {})
self.app.update_request(req)
res = controller.HEAD(req)
self.assertEquals(res.status[:len(str(expected))],
str(expected))
self.assertEquals(res.headers.get('last-modified'),
expected_timestamp)
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '2', '3'), '1')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '2'), '1')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '1'), '1')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3', '3', '1'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, '1', '2'), None)
def test_POST_meta_val_len(self):
with save_globals():
self.app.object_post_as_copy = False
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 200, 202, 202, 202)
# acct cont obj obj obj
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * 256})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 202)
proxy_server.http_connect = fake_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * 257})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
def test_POST_as_copy_meta_val_len(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
# acct cont objc objc objc obj obj obj
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * 256})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 202)
proxy_server.http_connect = fake_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x' * 257})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
def test_POST_meta_key_len(self):
with save_globals():
self.app.object_post_as_copy = False
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 200, 202, 202, 202)
# acct cont obj obj obj
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
('X-Object-Meta-' + 'x' * 128): 'x'})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 202)
proxy_server.http_connect = fake_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
('X-Object-Meta-' + 'x' * 129): 'x'})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
def test_POST_as_copy_meta_key_len(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
# acct cont objc objc objc obj obj obj
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
('X-Object-Meta-' + 'x' * 128): 'x'})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 202)
proxy_server.http_connect = fake_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
('X-Object-Meta-' + 'x' * 129): 'x'})
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
def test_POST_meta_count(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
headers = dict(
(('X-Object-Meta-' + str(i), 'a') for i in xrange(91)))
headers.update({'Content-Type': 'foo/bar'})
proxy_server.http_connect = fake_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers=headers)
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
def test_POST_meta_size(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
headers = dict(
(('X-Object-Meta-' + str(i), 'a' * 256) for i in xrange(1000)))
headers.update({'Content-Type': 'foo/bar'})
proxy_server.http_connect = fake_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers=headers)
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
def test_client_timeout(self):
with save_globals():
self.app.account_ring.get_nodes('account')
for dev in self.app.account_ring.devs.values():
dev['ip'] = '127.0.0.1'
dev['port'] = 1
self.app.container_ring.get_nodes('account')
for dev in self.app.container_ring.devs.values():
dev['ip'] = '127.0.0.1'
dev['port'] = 1
self.app.object_ring.get_nodes('account')
for dev in self.app.object_ring.devs.values():
dev['ip'] = '127.0.0.1'
dev['port'] = 1
class SlowBody():
def __init__(self):
self.sent = 0
def read(self, size=-1):
if self.sent < 4:
sleep(0.1)
self.sent += 1
return ' '
return ''
req = Request.blank('/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()},
headers={'Content-Length': '4', 'Content-Type': 'text/plain'})
self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 200, 201, 201, 201)
# acct cont obj obj obj
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
self.app.client_timeout = 0.1
req = Request.blank('/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()},
headers={'Content-Length': '4', 'Content-Type': 'text/plain'})
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(201, 201, 201)
# obj obj obj
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 408)
def test_client_disconnect(self):
with save_globals():
self.app.account_ring.get_nodes('account')
for dev in self.app.account_ring.devs.values():
dev['ip'] = '127.0.0.1'
dev['port'] = 1
self.app.container_ring.get_nodes('account')
for dev in self.app.container_ring.devs.values():
dev['ip'] = '127.0.0.1'
dev['port'] = 1
self.app.object_ring.get_nodes('account')
for dev in self.app.object_ring.devs.values():
dev['ip'] = '127.0.0.1'
dev['port'] = 1
class SlowBody():
def __init__(self):
self.sent = 0
def read(self, size=-1):
raise Exception('Disconnected')
req = Request.blank('/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()},
headers={'Content-Length': '4', 'Content-Type': 'text/plain'})
self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 200, 201, 201, 201)
# acct cont obj obj obj
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 499)
def test_node_read_timeout(self):
with save_globals():
self.app.account_ring.get_nodes('account')
for dev in self.app.account_ring.devs.values():
dev['ip'] = '127.0.0.1'
dev['port'] = 1
self.app.container_ring.get_nodes('account')
for dev in self.app.container_ring.devs.values():
dev['ip'] = '127.0.0.1'
dev['port'] = 1
self.app.object_ring.get_nodes('account')
for dev in self.app.object_ring.devs.values():
dev['ip'] = '127.0.0.1'
dev['port'] = 1
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, slow=True)
req.sent_size = 0
resp = controller.GET(req)
got_exc = False
try:
resp.body
except proxy_server.ChunkReadTimeout:
got_exc = True
self.assert_(not got_exc)
self.app.node_timeout = 0.1
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, slow=True)
resp = controller.GET(req)
got_exc = False
try:
resp.body
except proxy_server.ChunkReadTimeout:
got_exc = True
self.assert_(got_exc)
def test_node_write_timeout(self):
with save_globals():
self.app.account_ring.get_nodes('account')
for dev in self.app.account_ring.devs.values():
dev['ip'] = '127.0.0.1'
dev['port'] = 1
self.app.container_ring.get_nodes('account')
for dev in self.app.container_ring.devs.values():
dev['ip'] = '127.0.0.1'
dev['port'] = 1
self.app.object_ring.get_nodes('account')
for dev in self.app.object_ring.devs.values():
dev['ip'] = '127.0.0.1'
dev['port'] = 1
req = Request.blank('/a/c/o',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '4', 'Content-Type': 'text/plain'},
body=' ')
self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 200, 201, 201, 201, slow=True)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
self.app.node_timeout = 0.1
proxy_server.http_connect = \
fake_http_connect(201, 201, 201, slow=True)
req = Request.blank('/a/c/o',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '4', 'Content-Type': 'text/plain'},
body=' ')
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 503)
def test_iter_nodes(self):
with save_globals():
try:
self.app.object_ring.max_more_nodes = 2
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
partition, nodes = self.app.object_ring.get_nodes('account',
'container', 'object')
collected_nodes = []
for node in controller.iter_nodes(partition, nodes,
self.app.object_ring):
collected_nodes.append(node)
self.assertEquals(len(collected_nodes), 5)
self.app.object_ring.max_more_nodes = 20
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
partition, nodes = self.app.object_ring.get_nodes('account',
'container', 'object')
collected_nodes = []
for node in controller.iter_nodes(partition, nodes,
self.app.object_ring):
collected_nodes.append(node)
self.assertEquals(len(collected_nodes), 9)
finally:
self.app.object_ring.max_more_nodes = 0
def test_best_response_sets_etag(self):
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'})
resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3,
'Object')
self.assertEquals(resp.etag, None)
resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3,
'Object', etag='68b329da9893e34099c7d8ad5cb9c940')
self.assertEquals(resp.etag, '68b329da9893e34099c7d8ad5cb9c940')
def test_proxy_passes_content_type(self):
with save_globals():
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = fake_http_connect(200, 200, 200)
resp = controller.GET(req)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_type, 'x-application/test')
proxy_server.http_connect = fake_http_connect(200, 200, 200)
resp = controller.GET(req)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_length, 0)
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, slow=True)
resp = controller.GET(req)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_length, 4)
def test_proxy_passes_content_length_on_head(self):
with save_globals():
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'HEAD'})
self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = fake_http_connect(200, 200, 200)
resp = controller.HEAD(req)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_length, 0)
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, slow=True)
resp = controller.HEAD(req)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_length, 4)
def test_error_limiting(self):
with save_globals():
proxy_server.shuffle = lambda l: None
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
self.assert_status_map(controller.HEAD, (200, 200, 503, 200, 200), 200)
self.assertEquals(controller.app.object_ring.devs[0]['errors'], 2)
self.assert_('last_error' in controller.app.object_ring.devs[0])
for _junk in xrange(self.app.error_suppression_limit):
self.assert_status_map(controller.HEAD, (200, 200, 503, 503, 503), 503)
self.assertEquals(controller.app.object_ring.devs[0]['errors'],
self.app.error_suppression_limit + 1)
self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200), 503)
self.assert_('last_error' in controller.app.object_ring.devs[0])
self.assert_status_map(controller.PUT, (200, 200, 200, 201, 201, 201), 503)
self.assert_status_map(controller.POST,
(200, 200, 200, 200, 200, 200, 202, 202, 202), 503)
self.assert_status_map(controller.DELETE,
(200, 200, 200, 204, 204, 204), 503)
self.app.error_suppression_interval = -300
self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200), 200)
self.assertRaises(BaseException,
self.assert_status_map, controller.DELETE,
(200, 200, 200, 204, 204, 204), 503, raise_exc=True)
def test_acc_or_con_missing_returns_404(self):
with save_globals():
self.app.memcache = FakeMemcacheReturnsNone()
for dev in self.app.account_ring.devs.values():
del dev['errors']
del dev['last_error']
for dev in self.app.container_ring.devs.values():
del dev['errors']
del dev['last_error']
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 200)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
self.app.update_request(req)
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 200)
proxy_server.http_connect = \
fake_http_connect(404, 404, 404)
# acct acct acct
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
proxy_server.http_connect = \
fake_http_connect(503, 404, 404)
# acct acct acct
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
proxy_server.http_connect = \
fake_http_connect(503, 503, 404)
# acct acct acct
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
proxy_server.http_connect = \
fake_http_connect(503, 503, 503)
# acct acct acct
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
proxy_server.http_connect = \
fake_http_connect(200, 200, 204, 204, 204)
# acct cont obj obj obj
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 204)
proxy_server.http_connect = \
fake_http_connect(200, 404, 404, 404)
# acct cont cont cont
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
proxy_server.http_connect = \
fake_http_connect(200, 503, 503, 503)
# acct cont cont cont
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
for dev in self.app.account_ring.devs.values():
dev['errors'] = self.app.error_suppression_limit + 1
dev['last_error'] = time()
proxy_server.http_connect = \
fake_http_connect(200)
# acct [isn't actually called since everything
# is error limited]
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
for dev in self.app.account_ring.devs.values():
dev['errors'] = 0
for dev in self.app.container_ring.devs.values():
dev['errors'] = self.app.error_suppression_limit + 1
dev['last_error'] = time()
proxy_server.http_connect = \
fake_http_connect(200, 200)
# acct cont [isn't actually called since
# everything is error limited]
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 404)
def test_PUT_POST_requires_container_exist(self):
with save_globals():
self.app.object_post_as_copy = False
self.app.memcache = FakeMemcacheReturnsNone()
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 404, 404, 404, 200, 200, 200)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'})
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 404)
proxy_server.http_connect = \
fake_http_connect(200, 404, 404, 404, 200, 200)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
headers={'Content-Type': 'text/plain'})
self.app.update_request(req)
resp = controller.POST(req)
self.assertEquals(resp.status_int, 404)
def test_PUT_POST_as_copy_requires_container_exist(self):
with save_globals():
self.app.memcache = FakeMemcacheReturnsNone()
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 404, 404, 404, 200, 200, 200)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'})
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 404)
proxy_server.http_connect = \
fake_http_connect(200, 404, 404, 404, 200, 200, 200, 200, 200,
200)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
headers={'Content-Type': 'text/plain'})
self.app.update_request(req)
resp = controller.POST(req)
self.assertEquals(resp.status_int, 404)
def test_bad_metadata(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
fake_http_connect(200, 200, 201, 201, 201)
# acct cont obj obj obj
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0'})
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Object-Meta-' + ('a' *
MAX_META_NAME_LENGTH): 'v'})
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Object-Meta-' + ('a' *
(MAX_META_NAME_LENGTH + 1)): 'v'})
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 400)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Object-Meta-Too-Long': 'a' *
MAX_META_VALUE_LENGTH})
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Object-Meta-Too-Long': 'a' *
(MAX_META_VALUE_LENGTH + 1)})
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 400)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
headers = {'Content-Length': '0'}
for x in xrange(MAX_META_COUNT):
headers['X-Object-Meta-%d' % x] = 'v'
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers=headers)
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
headers = {'Content-Length': '0'}
for x in xrange(MAX_META_COUNT + 1):
headers['X-Object-Meta-%d' % x] = 'v'
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers=headers)
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 400)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
headers = {'Content-Length': '0'}
header_value = 'a' * MAX_META_VALUE_LENGTH
size = 0
x = 0
while size < MAX_META_OVERALL_SIZE - 4 - \
MAX_META_VALUE_LENGTH:
size += 4 + MAX_META_VALUE_LENGTH
headers['X-Object-Meta-%04d' % x] = header_value
x += 1
if MAX_META_OVERALL_SIZE - size > 1:
headers['X-Object-Meta-a'] = \
'a' * (MAX_META_OVERALL_SIZE - size - 1)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers=headers)
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
headers['X-Object-Meta-a'] = \
'a' * (MAX_META_OVERALL_SIZE - size)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers=headers)
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 400)
def test_copy_from(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
# initial source object PUT
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0'})
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 201, 201, 201)
# acct cont obj obj obj
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
# basic copy
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': 'c/o'})
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {}
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
self.assertEquals(resp.headers['x-copied-from'], 'c/o')
# non-zero content length
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '5',
'X-Copy-From': 'c/o'})
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 200, 200)
# acct cont acct cont objc objc objc
self.app.memcache.store = {}
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 400)
# extra source path parsing
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': 'c/o/o2'})
req.account = 'a'
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {}
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2')
# space in soure path
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': 'c/o%20o2'})
req.account = 'a'
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {}
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
self.assertEquals(resp.headers['x-copied-from'], 'c/o%20o2')
# repeat tests with leading /
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': '/c/o'})
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {}
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
self.assertEquals(resp.headers['x-copied-from'], 'c/o')
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': '/c/o/o2'})
req.account = 'a'
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {}
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2')
# negative tests
# invalid x-copy-from path
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': '/c'})
self.app.update_request(req)
self.app.memcache.store = {}
resp = controller.PUT(req)
self.assertEquals(resp.status_int // 100, 4) # client error
# server error
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': '/c/o'})
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 503, 503, 503)
# acct cont objc objc objc
self.app.memcache.store = {}
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 503)
# not found
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': '/c/o'})
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 404, 404, 404)
# acct cont objc objc objc
self.app.memcache.store = {}
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 404)
# some missing containers
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': '/c/o'})
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 404, 404, 200, 201, 201, 201)
# acct cont objc objc objc obj obj obj
self.app.memcache.store = {}
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
# test object meta data
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': '/c/o',
'X-Object-Meta-Ours': 'okay'})
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
# acct cont objc objc objc obj obj obj
self.app.memcache.store = {}
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
self.assertEquals(resp.headers.get('x-object-meta-test'),
'testing')
self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay')
def test_COPY(self):
with save_globals():
controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0'})
req.account = 'a'
proxy_server.http_connect = \
fake_http_connect(200, 200, 201, 201, 201)
# acct cont obj obj obj
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
headers={'Destination': 'c/o'})
req.account = 'a'
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {}
resp = controller.COPY(req)
self.assertEquals(resp.status_int, 201)
self.assertEquals(resp.headers['x-copied-from'], 'c/o')
req = Request.blank('/a/c/o/o2',
environ={'REQUEST_METHOD': 'COPY'},
headers={'Destination': 'c/o'})
req.account = 'a'
controller.object_name = 'o/o2'
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {}
resp = controller.COPY(req)
self.assertEquals(resp.status_int, 201)
self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2')
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
headers={'Destination': '/c/o'})
req.account = 'a'
controller.object_name = 'o'
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {}
resp = controller.COPY(req)
self.assertEquals(resp.status_int, 201)
self.assertEquals(resp.headers['x-copied-from'], 'c/o')
req = Request.blank('/a/c/o/o2',
environ={'REQUEST_METHOD': 'COPY'},
headers={'Destination': '/c/o'})
req.account = 'a'
controller.object_name = 'o/o2'
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
201)
# acct cont acct cont objc objc objc obj obj
# obj
self.app.memcache.store = {}
resp = controller.COPY(req)
self.assertEquals(resp.status_int, 201)
self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2')
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
headers={'Destination': 'c_o'})
req.account = 'a'
controller.object_name = 'o'
proxy_server.http_connect = \
fake_http_connect(200, 200)
# acct cont
self.app.memcache.store = {}
resp = controller.COPY(req)
self.assertEquals(resp.status_int, 412)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
headers={'Destination': '/c/o'})
req.account = 'a'
controller.object_name = 'o'
proxy_server.http_connect = \
fake_http_connect(200, 200, 503, 503, 503)
# acct cont objc objc objc
self.app.memcache.store = {}
resp = controller.COPY(req)
self.assertEquals(