openstack/swift
Code Issues Proposed changes
e5105ffa09
swift/test/unit/common/test_internal_client.py
Matthew Oliver e5105ffa09 internal_client: Remove allow_modify_pipeline option 
The internal client is suppose to be internal to the cluster, and as
such we rely on it to not remove any headers we decide to send. However
if the allow_modify_pipeline option is set the gatekeeper middleware is
added to the internal client's proxy pipeline.

So firstly, this patch removes the allow_modify_pipeline option from the
internal client constructor. And when calling loadapp
allow_modify_pipeline is always passed with a False.

Further, an op could directly put the gatekeeper middleware into the
internal client config. The internal client constructor will now check
the pipeline and raise a ValueError if one has been placed in the
pipeline.

To do this, there is now a check_gatekeeper_loaded staticmethod that will
walk the pipeline which called from the InternalClient.__init__ method.
Enabling this walking through the pipeline, we are now stashing the wsgi
pipeline in each filter so that we don't have to rely on 'app' naming
conventions to iterate the pipeline.

Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Change-Id: Idcca7ac0796935c8883de9084d612d64159d9f92
2023-04-14 10:37:40 +01:00

2064 lines
84 KiB
Python
Raw Blame History

# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import mock
import unittest
import zlib
import os
from io import BytesIO
from textwrap import dedent
import six
from six.moves import range, zip_longest
from six.moves.urllib.parse import quote, parse_qsl
from swift.common import exceptions, internal_client, request_helpers, swob, \
utils
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.storage_policy import StoragePolicy
from swift.common.middleware.proxy_logging import ProxyLoggingMiddleware
from swift.common.middleware.gatekeeper import GatekeeperMiddleware
from test.debug_logger import debug_logger
from test.unit import with_tempdir, write_fake_ring, patch_policies
from test.unit.common.middleware.helpers import FakeSwift, LeakTrackingIter
if six.PY3:
from eventlet.green.urllib import request as urllib2
else:
from eventlet.green import urllib2
class FakeConn(object):
def __init__(self, body=None):
if body is None:
body = []
self.body = body
def read(self):
return json.dumps(self.body).encode('ascii')
def info(self):
return {}
def not_sleep(seconds):
pass
def unicode_string(start, length):
return u''.join([six.unichr(x) for x in range(start, start + length)])
def path_parts():
account = unicode_string(1000, 4) + ' ' + unicode_string(1100, 4)
container = unicode_string(2000, 4) + ' ' + unicode_string(2100, 4)
obj = unicode_string(3000, 4) + ' ' + unicode_string(3100, 4)
return account, container, obj
def make_path(account, container=None, obj=None):
path = '/v1/%s' % quote(account.encode('utf-8'))
if container:
path += '/%s' % quote(container.encode('utf-8'))
if obj:
path += '/%s' % quote(obj.encode('utf-8'))
return path
def make_path_info(account, container=None, obj=None):
# FakeSwift keys on PATH_INFO - which is *encoded* but unquoted
path = '/v1/%s' % '/'.join(
p for p in (account, container, obj) if p)
return swob.bytes_to_wsgi(path.encode('utf-8'))
def get_client_app():
app = FakeSwift()
client = internal_client.InternalClient({}, 'test', 1, app=app)
return client, app
class InternalClient(internal_client.InternalClient):
def __init__(self):
pass
class GetMetadataInternalClient(internal_client.InternalClient):
def __init__(self, test, path, metadata_prefix, acceptable_statuses):
self.test = test
self.path = path
self.metadata_prefix = metadata_prefix
self.acceptable_statuses = acceptable_statuses
self.get_metadata_called = 0
self.metadata = 'some_metadata'
def _get_metadata(self, path, metadata_prefix, acceptable_statuses=None,
headers=None, params=None):
self.get_metadata_called += 1
self.test.assertEqual(self.path, path)
self.test.assertEqual(self.metadata_prefix, metadata_prefix)
self.test.assertEqual(self.acceptable_statuses, acceptable_statuses)
return self.metadata
class SetMetadataInternalClient(internal_client.InternalClient):
def __init__(
self, test, path, metadata, metadata_prefix, acceptable_statuses):
self.test = test
self.path = path
self.metadata = metadata
self.metadata_prefix = metadata_prefix
self.acceptable_statuses = acceptable_statuses
self.set_metadata_called = 0
self.metadata = 'some_metadata'
def _set_metadata(
self, path, metadata, metadata_prefix='',
acceptable_statuses=None):
self.set_metadata_called += 1
self.test.assertEqual(self.path, path)
self.test.assertEqual(self.metadata_prefix, metadata_prefix)
self.test.assertEqual(self.metadata, metadata)
self.test.assertEqual(self.acceptable_statuses, acceptable_statuses)
class IterInternalClient(internal_client.InternalClient):
def __init__(
self, test, path, marker, end_marker, prefix, acceptable_statuses,
items):
self.test = test
self.path = path
self.marker = marker
self.end_marker = end_marker
self.prefix = prefix
self.acceptable_statuses = acceptable_statuses
self.items = items
def _iter_items(
self, path, marker='', end_marker='', prefix='',
acceptable_statuses=None):
self.test.assertEqual(self.path, path)
self.test.assertEqual(self.marker, marker)
self.test.assertEqual(self.end_marker, end_marker)
self.test.assertEqual(self.prefix, prefix)
self.test.assertEqual(self.acceptable_statuses, acceptable_statuses)
for item in self.items:
yield item
class TestCompressingfileReader(unittest.TestCase):
def test_init(self):
class CompressObj(object):
def __init__(self, test, *args):
self.test = test
self.args = args
def method(self, *args):
self.test.assertEqual(self.args, args)
return self
try:
compressobj = CompressObj(
self, 9, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)
old_compressobj = internal_client.compressobj
internal_client.compressobj = compressobj.method
f = BytesIO(b'')
fobj = internal_client.CompressingFileReader(f)
self.assertEqual(f, fobj._f)
self.assertEqual(compressobj, fobj._compressor)
self.assertEqual(False, fobj.done)
self.assertEqual(True, fobj.first)
self.assertEqual(0, fobj.crc32)
self.assertEqual(0, fobj.total_size)
finally:
internal_client.compressobj = old_compressobj
def test_read(self):
exp_data = b'abcdefghijklmnopqrstuvwxyz'
fobj = internal_client.CompressingFileReader(
BytesIO(exp_data), chunk_size=5)
d = zlib.decompressobj(16 + zlib.MAX_WBITS)
data = b''.join(d.decompress(chunk)
for chunk in iter(fobj.read, b''))
self.assertEqual(exp_data, data)
def test_seek(self):
exp_data = b'abcdefghijklmnopqrstuvwxyz'
fobj = internal_client.CompressingFileReader(
BytesIO(exp_data), chunk_size=5)
# read a couple of chunks only
for _ in range(2):
fobj.read()
# read whole thing after seek and check data
fobj.seek(0)
d = zlib.decompressobj(16 + zlib.MAX_WBITS)
data = b''.join(d.decompress(chunk)
for chunk in iter(fobj.read, b''))
self.assertEqual(exp_data, data)
def test_seek_not_implemented_exception(self):
fobj = internal_client.CompressingFileReader(
BytesIO(b''), chunk_size=5)
self.assertRaises(NotImplementedError, fobj.seek, 10)
self.assertRaises(NotImplementedError, fobj.seek, 0, 10)
class TestInternalClient(unittest.TestCase):
@mock.patch('swift.common.utils.HASH_PATH_SUFFIX', new=b'endcap')
@with_tempdir
def test_load_from_config(self, tempdir):
conf_path = os.path.join(tempdir, 'interal_client.conf')
conf_body = """
[DEFAULT]
swift_dir = %s
[pipeline:main]
pipeline = catch_errors cache proxy-server
[app:proxy-server]
use = egg:swift#proxy
auto_create_account_prefix = -
[filter:cache]
use = egg:swift#memcache
[filter:catch_errors]
use = egg:swift#catch_errors
""" % tempdir
with open(conf_path, 'w') as f:
f.write(dedent(conf_body))
account_ring_path = os.path.join(tempdir, 'account.ring.gz')
write_fake_ring(account_ring_path)
container_ring_path = os.path.join(tempdir, 'container.ring.gz')
write_fake_ring(container_ring_path)
object_ring_path = os.path.join(tempdir, 'object.ring.gz')
write_fake_ring(object_ring_path)
logger = debug_logger('test-ic')
self.assertEqual(logger.get_lines_for_level('warning'), [])
with patch_policies([StoragePolicy(0, 'legacy', True)]):
with mock.patch('swift.proxy.server.get_logger',
lambda *a, **kw: logger):
client = internal_client.InternalClient(conf_path, 'test', 1)
self.assertEqual(logger.get_lines_for_level('warning'), [
'Option auto_create_account_prefix is deprecated. '
'Configure auto_create_account_prefix under the '
'swift-constraints section of swift.conf. This option will '
'be ignored in a future release.'])
self.assertEqual(client.account_ring,
client.app.app.app.account_ring)
self.assertEqual(client.account_ring.serialized_path,
account_ring_path)
self.assertEqual(client.container_ring,
client.app.app.app.container_ring)
self.assertEqual(client.container_ring.serialized_path,
container_ring_path)
object_ring = client.app.app.app.get_object_ring(0)
self.assertEqual(client.get_object_ring(0),
object_ring)
self.assertEqual(object_ring.serialized_path,
object_ring_path)
self.assertEqual(client.auto_create_account_prefix, '-')
@mock.patch('swift.common.utils.HASH_PATH_SUFFIX', new=b'endcap')
@with_tempdir
def test_load_from_config_with_global_conf(self, tempdir):
account_ring_path = os.path.join(tempdir, 'account.ring.gz')
write_fake_ring(account_ring_path)
container_ring_path = os.path.join(tempdir, 'container.ring.gz')
write_fake_ring(container_ring_path)
object_ring_path = os.path.join(tempdir, 'object.ring.gz')
write_fake_ring(object_ring_path)
# global_conf will override the 'x = y' syntax in conf file...
conf_path = os.path.join(tempdir, 'internal_client.conf')
conf_body = """
[DEFAULT]
swift_dir = %s
log_name = conf-file-log-name
[pipeline:main]
pipeline = catch_errors cache proxy-server
[app:proxy-server]
use = egg:swift#proxy
auto_create_account_prefix = -
[filter:cache]
use = egg:swift#memcache
[filter:catch_errors]
use = egg:swift#catch_errors
log_name = catch-errors-log-name
""" % tempdir
with open(conf_path, 'w') as f:
f.write(dedent(conf_body))
global_conf = {'log_name': 'global-conf-log-name'}
with patch_policies([StoragePolicy(0, 'legacy', True)]):
client = internal_client.InternalClient(
conf_path, 'test', 1, global_conf=global_conf)
self.assertEqual('global-conf-log-name', client.app.logger.server)
# ...but the 'set x = y' syntax in conf file DEFAULT section will
# override global_conf
conf_body = """
[DEFAULT]
swift_dir = %s
set log_name = conf-file-log-name
[pipeline:main]
pipeline = catch_errors cache proxy-server
[app:proxy-server]
use = egg:swift#proxy
auto_create_account_prefix = -
[filter:cache]
use = egg:swift#memcache
[filter:catch_errors]
use = egg:swift#catch_errors
log_name = catch-errors-log-name
""" % tempdir
with open(conf_path, 'w') as f:
f.write(dedent(conf_body))
global_conf = {'log_name': 'global-conf-log-name'}
with patch_policies([StoragePolicy(0, 'legacy', True)]):
client = internal_client.InternalClient(
conf_path, 'test', 1, global_conf=global_conf)
self.assertEqual('conf-file-log-name', client.app.logger.server)
# ...and the 'set x = y' syntax in conf file app section will override
# DEFAULT section and global_conf
conf_body = """
[DEFAULT]
swift_dir = %s
set log_name = conf-file-log-name
[pipeline:main]
pipeline = catch_errors cache proxy-server
[app:proxy-server]
use = egg:swift#proxy
auto_create_account_prefix = -
[filter:cache]
use = egg:swift#memcache
[filter:catch_errors]
use = egg:swift#catch_errors
set log_name = catch-errors-log-name
""" % tempdir
with open(conf_path, 'w') as f:
f.write(dedent(conf_body))
global_conf = {'log_name': 'global-conf-log-name'}
with patch_policies([StoragePolicy(0, 'legacy', True)]):
client = internal_client.InternalClient(
conf_path, 'test', 1, global_conf=global_conf)
self.assertEqual('catch-errors-log-name', client.app.logger.server)
def test_init(self):
conf_path = 'some_path'
app = FakeSwift()
user_agent = 'some_user_agent'
request_tries = 123
with mock.patch.object(
internal_client, 'loadapp', return_value=app) as mock_loadapp,\
self.assertRaises(ValueError):
# First try with a bad arg
internal_client.InternalClient(
conf_path, user_agent, request_tries=0)
mock_loadapp.assert_not_called()
# if we load it with the gatekeeper middleware then we also get a
# value error
gate_keeper_app = GatekeeperMiddleware(app, {})
gate_keeper_app._pipeline_final_app = app
gate_keeper_app._pipeline = [gate_keeper_app, app]
with mock.patch.object(
internal_client, 'loadapp', return_value=gate_keeper_app) \
as mock_loadapp, self.assertRaises(ValueError) as err:
internal_client.InternalClient(
conf_path, user_agent, request_tries)
self.assertEqual(
str(err.exception),
('Gatekeeper middleware is not allowed in the InternalClient '
'proxy pipeline'))
with mock.patch.object(
internal_client, 'loadapp', return_value=app) as mock_loadapp:
client = internal_client.InternalClient(
conf_path, user_agent, request_tries)
mock_loadapp.assert_called_once_with(
conf_path, global_conf=None, allow_modify_pipeline=False)
self.assertEqual(app, client.app)
self.assertEqual(user_agent, client.user_agent)
self.assertEqual(request_tries, client.request_tries)
self.assertFalse(client.use_replication_network)
client = internal_client.InternalClient(
conf_path, user_agent, request_tries, app=app,
use_replication_network=True)
self.assertEqual(app, client.app)
self.assertEqual(user_agent, client.user_agent)
self.assertEqual(request_tries, client.request_tries)
self.assertTrue(client.use_replication_network)
global_conf = {'log_name': 'custom'}
client = internal_client.InternalClient(
conf_path, user_agent, request_tries, app=app,
use_replication_network=True, global_conf=global_conf)
self.assertEqual(app, client.app)
self.assertEqual(user_agent, client.user_agent)
self.assertEqual(request_tries, client.request_tries)
self.assertTrue(client.use_replication_network)
def test_gatekeeper_not_loaded(self):
app = FakeSwift()
pipeline = [app]
class RandomMiddleware(object):
def __init__(self, app):
self.app = app
self._pipeline_final_app = app
self._pipeline = pipeline
self._pipeline.insert(0, self)
# if there is no Gatekeeper middleware then it's false
# just the final app
self.assertFalse(
internal_client.InternalClient.check_gatekeeper_not_loaded(app))
# now with a bunch of middlewares
app_no_gatekeeper = app
for i in range(5):
app_no_gatekeeper = RandomMiddleware(app_no_gatekeeper)
self.assertFalse(
internal_client.InternalClient.check_gatekeeper_not_loaded(
app_no_gatekeeper))
# But if we put the gatekeeper on the end, it will be found
app_with_gatekeeper = GatekeeperMiddleware(app_no_gatekeeper, {})
pipeline.insert(0, app_with_gatekeeper)
app_with_gatekeeper._pipeline = pipeline
with self.assertRaises(ValueError) as err:
internal_client.InternalClient.check_gatekeeper_not_loaded(
app_with_gatekeeper)
self.assertEqual(str(err.exception),
('Gatekeeper middleware is not allowed in the '
'InternalClient proxy pipeline'))
# even if we bury deep into the pipeline
for i in range(5):
app_with_gatekeeper = RandomMiddleware(app_with_gatekeeper)
with self.assertRaises(ValueError) as err:
internal_client.InternalClient.check_gatekeeper_not_loaded(
app_with_gatekeeper)
self.assertEqual(str(err.exception),
('Gatekeeper middleware is not allowed in the '
'InternalClient proxy pipeline'))
def test_make_request_sets_user_agent(self):
class FakeApp(FakeSwift):
def __init__(self, test):
super(FakeApp, self).__init__()
self.test = test
def __call__(self, env, start_response):
self.test.assertNotIn(
'HTTP_X_BACKEND_USE_REPLICATION_NETWORK', env)
self.test.assertEqual(self.backend_user_agent,
"some_agent")
start_response('200 Ok', [('Content-Length', '0')])
return []
client = internal_client.InternalClient(
None, 'some_agent', 1, use_replication_network=False,
app=FakeApp(self))
client.make_request('GET', '/', {}, (200,))
def test_make_request_clears_txn_id_after_calling_app(self):
class InternalClient(internal_client.InternalClient):
def __init__(self, test, logger):
def fake_app(env, start_response):
self.app.logger.txn_id = 'foo'
self.app.logger.debug('Inside of request')
start_response('200 Ok', [('Content-Length', '0')])
return []
self.test = test
self.user_agent = 'some_agent'
self.app = fake_app
self.app.logger = logger
self.request_tries = 1
self.use_replication_network = False
fake_logger = debug_logger()
logger = utils.LogAdapter(fake_logger, 'test-server')
# Make sure there's no transaction ID set -- other tests may have
# polluted the logger
logger.txn_id = None
logger.debug('Before request')
client = InternalClient(self, logger)
client.make_request('GET', '/', {}, (200,))
logger.debug('After request')
self.assertEqual([(args[0], kwargs['extra'].get('txn_id'))
for args, kwargs in fake_logger.log_dict['debug']], [
('Before request', None),
('Inside of request', 'foo'),
('After request', None),
])
def test_make_request_defaults_replication_network_header(self):
class FakeApp(FakeSwift):
def __init__(self, test):
super(FakeApp, self).__init__()
self.test = test
self.expected_header_value = None
def __call__(self, env, start_response):
if self.expected_header_value is None:
self.test.assertNotIn(
'HTTP_X_BACKEND_USE_REPLICATION_NETWORK', env)
else:
hdr_val = env['HTTP_X_BACKEND_USE_REPLICATION_NETWORK']
self.test.assertEqual(self.expected_header_value, hdr_val)
self.test.assertEqual(self.backend_user_agent,
'some_agent')
start_response('200 Ok', [('Content-Length', '0')])
return []
client = internal_client.InternalClient(
None, 'some_agent', 1, use_replication_network=False,
app=FakeApp(self))
client.make_request('GET', '/', {}, (200,))
# Caller can still override
client.app.expected_header_value = 'false'
client.make_request('GET', '/', {
request_helpers.USE_REPLICATION_NETWORK_HEADER: 'false'}, (200,))
client.app.expected_header_value = 'true'
client.make_request('GET', '/', {
request_helpers.USE_REPLICATION_NETWORK_HEADER: 'true'}, (200,))
# Switch default behavior
client.use_replication_network = True
client.make_request('GET', '/', {}, (200,))
client.app.expected_header_value = 'false'
client.make_request('GET', '/', {
request_helpers.USE_REPLICATION_NETWORK_HEADER: 'false'}, (200,))
client.app.expected_header_value = 'on'
client.make_request('GET', '/', {
request_helpers.USE_REPLICATION_NETWORK_HEADER: 'on'}, (200,))
def test_make_request_sets_query_string(self):
captured_envs = []
class FakeApp(FakeSwift):
def __call__(self, env, start_response):
captured_envs.append(env)
start_response('200 Ok', [('Content-Length', '0')])
return []
client = internal_client.InternalClient(
None, 'some_agent', 1, use_replication_network=False,
app=FakeApp())
params = {'param1': 'p1', 'tasty': 'soup'}
client.make_request('GET', '/', {}, (200,), params=params)
actual_params = dict(parse_qsl(captured_envs[0]['QUERY_STRING'],
keep_blank_values=True,
strict_parsing=True))
self.assertEqual(params, actual_params)
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
def test_make_request_retries(self):
class FakeApp(FakeSwift):
def __init__(self):
super(FakeApp, self).__init__()
self.request_tries = 4
self.tries = 0
def __call__(self, env, start_response):
self.tries += 1
if self.tries < self.request_tries:
start_response(
'500 Internal Server Error', [('Content-Length', '0')])
else:
start_response('200 Ok', [('Content-Length', '0')])
return []
class InternalClient(internal_client.InternalClient):
def __init__(self, *args, **kwargs):
self.test = kwargs.pop('test')
super(InternalClient, self).__init__(*args, **kwargs)
self.sleep_called = 0
def sleep(self, seconds):
self.sleep_called += 1
self.test.assertEqual(2 ** (self.sleep_called), seconds)
client = InternalClient(
None, 'some_agent', 4, use_replication_network=False,
app=FakeApp(), test=self)
old_sleep = internal_client.sleep
internal_client.sleep = client.sleep
try:
client.make_request('GET', '/', {}, (200,))
finally:
internal_client.sleep = old_sleep
self.assertEqual(3, client.sleep_called)
self.assertEqual(4, client.app.tries)
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
def test_base_request_timeout(self):
# verify that base_request passes timeout arg on to urlopen
body = {"some": "content"}
for timeout in (0.0, 42.0, None):
mocked_func = 'swift.common.internal_client.urllib2.urlopen'
with mock.patch(mocked_func) as mock_urlopen:
mock_urlopen.side_effect = [FakeConn(body)]
sc = internal_client.SimpleClient('http://0.0.0.0/')
_, resp_body = sc.base_request('GET', timeout=timeout)
mock_urlopen.assert_called_once_with(mock.ANY, timeout=timeout)
# sanity check
self.assertEqual(body, resp_body)
def test_base_full_listing(self):
body1 = [{'name': 'a'}, {'name': "b"}, {'name': "c"}]
body2 = [{'name': 'd'}]
body3 = []
mocked_func = 'swift.common.internal_client.urllib2.urlopen'
with mock.patch(mocked_func) as mock_urlopen:
mock_urlopen.side_effect = [
FakeConn(body1), FakeConn(body2), FakeConn(body3)]
sc = internal_client.SimpleClient('http://0.0.0.0/')
_, resp_body = sc.base_request('GET', full_listing=True)
self.assertEqual(body1 + body2, resp_body)
self.assertEqual(3, mock_urlopen.call_count)
actual_requests = [call[0][0] for call in mock_urlopen.call_args_list]
if six.PY2:
# The get_selector method was deprecated in favor of a selector
# attribute in py31 and removed in py34
self.assertEqual(
'/?format=json', actual_requests[0].get_selector())
self.assertEqual(
'/?format=json&marker=c', actual_requests[1].get_selector())
self.assertEqual(
'/?format=json&marker=d', actual_requests[2].get_selector())
else:
self.assertEqual('/?format=json', actual_requests[0].selector)
self.assertEqual(
'/?format=json&marker=c', actual_requests[1].selector)
self.assertEqual(
'/?format=json&marker=d', actual_requests[2].selector)
def test_make_request_method_path_headers(self):
class FakeApp(FakeSwift):
def __init__(self):
super(FakeApp, self).__init__()
self.env = None
def __call__(self, env, start_response):
self.env = env
start_response('200 Ok', [('Content-Length', '0')])
return []
client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp())
for method in 'GET PUT HEAD'.split():
client.make_request(method, '/', {}, (200,))
self.assertEqual(client.app.env['REQUEST_METHOD'], method)
for path in '/one /two/three'.split():
client.make_request('GET', path, {'X-Test': path}, (200,))
self.assertEqual(client.app.env['PATH_INFO'], path)
self.assertEqual(client.app.env['HTTP_X_TEST'], path)
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
def test_make_request_error_case(self):
class FakeApp(FakeSwift):
def __call__(self, env, start_response):
body = b'fake error response'
start_response('409 Conflict',
[('Content-Length', str(len(body)))])
return [body]
final_fake_app = FakeApp()
fake_app = ProxyLoggingMiddleware(
final_fake_app, {}, final_fake_app.logger)
fake_app._pipeline_final_app = final_fake_app
client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False, app=fake_app)
with self.assertRaises(internal_client.UnexpectedResponse), \
mock.patch('swift.common.internal_client.sleep'):
client.make_request('DELETE', '/container', {}, (200,))
# Since we didn't provide an X-Timestamp, retrying gives us a chance to
# succeed (assuming the failure was due to clock skew between servers)
expected = (' HTTP/1.0 409 ',)
logger = client.app._pipeline_final_app.logger
loglines = logger.get_lines_for_level('info')
for expected, logline in zip_longest(expected, loglines):
if not expected:
self.fail('Unexpected extra log line: %r' % logline)
self.assertIn(expected, logline)
self.assertEqual(client.app.app.backend_user_agent, 'some_agent')
def test_make_request_acceptable_status_not_2xx(self):
class FakeApp(FakeSwift):
def __init__(self):
super(FakeApp, self).__init__()
self.closed_paths = []
self.fully_read_paths = []
self.resp_status = None
def __call__(self, env, start_response):
body = b'fake error response'
start_response(self.resp_status,
[('Content-Length', str(len(body)))])
return LeakTrackingIter(body, self.closed_paths.append,
self.fully_read_paths.append,
env['PATH_INFO'])
def do_test(resp_status):
final_fake_app = FakeApp()
fake_app = ProxyLoggingMiddleware(
final_fake_app, {}, final_fake_app.logger)
fake_app._pipeline_final_app = final_fake_app
final_fake_app.resp_status = resp_status
client = internal_client.InternalClient(
None, "some_agent", 3, use_replication_network=False,
app=fake_app)
with self.assertRaises(internal_client.UnexpectedResponse) as ctx,\
mock.patch('swift.common.internal_client.sleep'):
# This is obvious strange tests to expect only 400 Bad Request
# but this test intended to avoid extra body drain if it's
# correct object body with 2xx.
client.make_request('GET', '/cont/obj', {}, (400,))
logger = client.app._pipeline_final_app.logger
loglines = logger.get_lines_for_level('info')
self.assertEqual(client.app.app.backend_user_agent, 'some_agent')
return (client.app._pipeline_final_app.fully_read_paths,
client.app._pipeline_final_app.closed_paths,
ctx.exception.resp, loglines)
fully_read_paths, closed_paths, resp, loglines = do_test('200 OK')
# Since the 200 is considered "properly handled", it won't be retried
self.assertEqual(fully_read_paths, [])
self.assertEqual(closed_paths, [])
# ...and it'll be on us (the caller) to read and close (for example,
# by using swob.Response's body property)
self.assertEqual(resp.body, b'fake error response')
self.assertEqual(fully_read_paths, ['/cont/obj'])
self.assertEqual(closed_paths, ['/cont/obj'])
expected = (' HTTP/1.0 200 ', )
for expected, logline in zip_longest(expected, loglines):
if not expected:
self.fail('Unexpected extra log line: %r' % logline)
self.assertIn(expected, logline)
fully_read_paths, closed_paths, resp, loglines = do_test(
'503 Service Unavailable')
# But since 5xx is neither "properly handled" not likely to include
# a large body, it will be retried and responses will already be closed
self.assertEqual(fully_read_paths, ['/cont/obj'] * 3)
self.assertEqual(closed_paths, ['/cont/obj'] * 3)
expected = (' HTTP/1.0 503 ', ' HTTP/1.0 503 ', ' HTTP/1.0 503 ', )
for expected, logline in zip_longest(expected, loglines):
if not expected:
self.fail('Unexpected extra log line: %r' % logline)
self.assertIn(expected, logline)
def test_make_request_codes(self):
class FakeApp(FakeSwift):
def __call__(self, env, start_response):
start_response('200 Ok', [('Content-Length', '0')])
return []
client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp())
try:
old_sleep = internal_client.sleep
internal_client.sleep = not_sleep
client.make_request('GET', '/', {}, (200,))
client.make_request('GET', '/', {}, (2,))
client.make_request('GET', '/', {}, (400, 200))
client.make_request('GET', '/', {}, (400, 2))
with self.assertRaises(internal_client.UnexpectedResponse) \
as raised:
client.make_request('GET', '/', {}, (400,))
self.assertEqual(200, raised.exception.resp.status_int)
with self.assertRaises(internal_client.UnexpectedResponse) \
as raised:
client.make_request('GET', '/', {}, (201,))
self.assertEqual(200, raised.exception.resp.status_int)
with self.assertRaises(internal_client.UnexpectedResponse) \
as raised:
client.make_request('GET', '/', {}, (111,))
self.assertTrue(str(raised.exception).startswith(
'Unexpected response'))
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
finally:
internal_client.sleep = old_sleep
def test_make_request_calls_fobj_seek_each_try(self):
class FileObject(object):
def __init__(self, test):
self.test = test
self.seek_called = 0
def seek(self, offset, whence=0):
self.seek_called += 1
self.test.assertEqual(0, offset)
self.test.assertEqual(0, whence)
class FakeApp(FakeSwift):
def __init__(self, status):
super(FakeApp, self).__init__()
self.status = status
def __call__(self, env, start_response):
start_response(self.status, [('Content-Length', '0')])
self._calls.append('')
return []
def do_test(status, expected_calls):
fobj = FileObject(self)
client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp(status))
with mock.patch.object(internal_client, 'sleep', not_sleep):
with self.assertRaises(Exception) as exc_mgr:
client.make_request('PUT', '/', {}, (2,), fobj)
self.assertEqual(int(status[:3]),
exc_mgr.exception.resp.status_int)
self.assertEqual(client.app.call_count, fobj.seek_called)
self.assertEqual(client.app.call_count, expected_calls)
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
do_test('404 Not Found', 1)
do_test('503 Service Unavailable', 3)
def test_make_request_request_exception(self):
class FakeApp(FakeSwift):
def __call__(self, env, start_response):
raise Exception()
client = internal_client.InternalClient(
None, 'some_agent', 3, app=FakeApp())
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
try:
old_sleep = internal_client.sleep
internal_client.sleep = not_sleep
self.assertRaises(
Exception, client.make_request, 'GET', '/', {}, (2,))
finally:
internal_client.sleep = old_sleep
def test_get_metadata(self):
class Response(object):
def __init__(self, headers):
self.headers = headers
self.status_int = 200
class InternalClient(internal_client.InternalClient):
def __init__(self, test, path, resp_headers):
self.test = test
self.path = path
self.resp_headers = resp_headers
self.make_request_called = 0
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None, params=None):
self.make_request_called += 1
self.test.assertEqual('HEAD', method)
self.test.assertEqual(self.path, path)
self.test.assertEqual((2,), acceptable_statuses)
self.test.assertIsNone(body_file)
return Response(self.resp_headers)
path = 'some_path'
metadata_prefix = 'some_key-'
resp_headers = {
'%sone' % (metadata_prefix): '1',
'%sTwo' % (metadata_prefix): '2',
'%sThree' % (metadata_prefix): '3',
'some_header-four': '4',
'Some_header-five': '5',
}
exp_metadata = {
'one': '1',
'two': '2',
'three': '3',
}
client = InternalClient(self, path, resp_headers)
metadata = client._get_metadata(path, metadata_prefix)
self.assertEqual(exp_metadata, metadata)
self.assertEqual(1, client.make_request_called)
def test_get_metadata_invalid_status(self):
class FakeApp(FakeSwift):
def __call__(self, environ, start_response):
start_response('404 Not Found', [('x-foo', 'bar')])
return [b'nope']
client = internal_client.InternalClient(
None, 'test', 1, use_replication_network=False, app=FakeApp())
self.assertRaises(internal_client.UnexpectedResponse,
client._get_metadata, 'path')
metadata = client._get_metadata('path', metadata_prefix='x-',
acceptable_statuses=(4,))
self.assertEqual(metadata, {'foo': 'bar'})
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'test')
def test_make_path(self):
account, container, obj = path_parts()
path = make_path(account, container, obj)
c = InternalClient()
self.assertEqual(path, c.make_path(account, container, obj))
def test_make_path_exception(self):
c = InternalClient()
self.assertRaises(ValueError, c.make_path, 'account', None, 'obj')
def test_iter_items(self):
class Response(object):
def __init__(self, status_int, body):
self.status_int = status_int
self.body = body
class InternalClient(internal_client.InternalClient):
def __init__(self, test, responses):
self.test = test
self.responses = responses
self.make_request_called = 0
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
self.make_request_called += 1
return self.responses.pop(0)
exp_items = []
responses = [Response(200, json.dumps([]).encode('ascii')), ]
items = []
client = InternalClient(self, responses)
for item in client._iter_items('/'):
items.append(item)
self.assertEqual(exp_items, items)
exp_items = []
responses = []
for i in range(3):
data = [
{'name': 'item%02d' % (2 * i)},
{'name': 'item%02d' % (2 * i + 1)}]
responses.append(Response(200, json.dumps(data).encode('ascii')))
exp_items.extend(data)
responses.append(Response(204, ''))
items = []
client = InternalClient(self, responses)
for item in client._iter_items('/'):
items.append(item)
self.assertEqual(exp_items, items)
def test_iter_items_with_markers(self):
class Response(object):
def __init__(self, status_int, body):
self.status_int = status_int
self.body = body.encode('ascii')
class InternalClient(internal_client.InternalClient):
def __init__(self, test, paths, responses):
self.test = test
self.paths = paths
self.responses = responses
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
exp_path = self.paths.pop(0)
self.test.assertEqual(exp_path, path)
return self.responses.pop(0)
paths = [
'/?format=json&marker=start&end_marker=end&prefix=',
'/?format=json&marker=one%C3%A9&end_marker=end&prefix=',
'/?format=json&marker=two&end_marker=end&prefix=',
]
responses = [
Response(200, json.dumps([{
'name': b'one\xc3\xa9'.decode('utf8')}, ])),
Response(200, json.dumps([{'name': 'two'}, ])),
Response(204, ''),
]
items = []
client = InternalClient(self, paths, responses)
for item in client._iter_items('/', marker='start', end_marker='end'):
items.append(item['name'].encode('utf8'))
self.assertEqual(b'one\xc3\xa9 two'.split(), items)
def test_iter_items_with_markers_and_prefix(self):
class Response(object):
def __init__(self, status_int, body):
self.status_int = status_int
self.body = body.encode('ascii')
class InternalClient(internal_client.InternalClient):
def __init__(self, test, paths, responses):
self.test = test
self.paths = paths
self.responses = responses
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
exp_path = self.paths.pop(0)
self.test.assertEqual(exp_path, path)
return self.responses.pop(0)
paths = [
'/?format=json&marker=prefixed_start&end_marker=prefixed_end'
'&prefix=prefixed_',
'/?format=json&marker=prefixed_one%C3%A9&end_marker=prefixed_end'
'&prefix=prefixed_',
'/?format=json&marker=prefixed_two&end_marker=prefixed_end'
'&prefix=prefixed_',
]
responses = [
Response(200, json.dumps([{
'name': b'prefixed_one\xc3\xa9'.decode('utf8')}, ])),
Response(200, json.dumps([{'name': 'prefixed_two'}, ])),
Response(204, ''),
]
items = []
client = InternalClient(self, paths, responses)
for item in client._iter_items('/', marker='prefixed_start',
end_marker='prefixed_end',
prefix='prefixed_'):
items.append(item['name'].encode('utf8'))
self.assertEqual(b'prefixed_one\xc3\xa9 prefixed_two'.split(), items)
def test_iter_item_read_response_if_status_is_acceptable(self):
class Response(object):
def __init__(self, status_int, body, app_iter):
self.status_int = status_int
self.body = body
self.app_iter = app_iter
class InternalClient(internal_client.InternalClient):
def __init__(self, test, responses):
self.test = test
self.responses = responses
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
resp = self.responses.pop(0)
if resp.status_int in acceptable_statuses or \
resp.status_int // 100 in acceptable_statuses:
return resp
if resp:
raise internal_client.UnexpectedResponse(
'Unexpected response: %s' % resp.status_int, resp)
num_list = []
def generate_resp_body():
for i in range(1, 5):
yield str(i).encode('ascii')
num_list.append(i)
exp_items = []
responses = [Response(204, json.dumps([]).encode('ascii'),
generate_resp_body())]
items = []
client = InternalClient(self, responses)
for item in client._iter_items('/'):
items.append(item)
self.assertEqual(exp_items, items)
self.assertEqual(len(num_list), 0)
responses = [Response(300, json.dumps([]).encode('ascii'),
generate_resp_body())]
client = InternalClient(self, responses)
self.assertRaises(internal_client.UnexpectedResponse,
next, client._iter_items('/'))
exp_items = []
responses = [Response(404, json.dumps([]).encode('ascii'),
generate_resp_body())]
items = []
client = InternalClient(self, responses)
for item in client._iter_items('/'):
items.append(item)
self.assertEqual(exp_items, items)
self.assertEqual(len(num_list), 4)
def test_set_metadata(self):
class InternalClient(internal_client.InternalClient):
def __init__(self, test, path, exp_headers):
self.test = test
self.path = path
self.exp_headers = exp_headers
self.make_request_called = 0
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
self.make_request_called += 1
self.test.assertEqual('POST', method)
self.test.assertEqual(self.path, path)
self.test.assertEqual(self.exp_headers, headers)
self.test.assertEqual((2,), acceptable_statuses)
self.test.assertIsNone(body_file)
path = 'some_path'
metadata_prefix = 'some_key-'
metadata = {
'%sone' % (metadata_prefix): '1',
'%stwo' % (metadata_prefix): '2',
'three': '3',
}
exp_headers = {
'%sone' % (metadata_prefix): '1',
'%stwo' % (metadata_prefix): '2',
'%sthree' % (metadata_prefix): '3',
}
client = InternalClient(self, path, exp_headers)
client._set_metadata(path, metadata, metadata_prefix)
self.assertEqual(1, client.make_request_called)
def test_iter_containers(self):
account, container, obj = path_parts()
path = make_path(account)
items = '0 1 2'.split()
marker = 'some_marker'
end_marker = 'some_end_marker'
prefix = 'some_prefix'
acceptable_statuses = 'some_status_list'
client = IterInternalClient(
self, path, marker, end_marker, prefix, acceptable_statuses, items)
ret_items = []
for container in client.iter_containers(
account, marker, end_marker, prefix,
acceptable_statuses=acceptable_statuses):
ret_items.append(container)
self.assertEqual(items, ret_items)
def test_create_account(self):
account, container, obj = path_parts()
path = make_path_info(account)
client, app = get_client_app()
app.register('PUT', path, swob.HTTPCreated, {})
client.create_account(account)
self.assertEqual([('PUT', path, {
'X-Backend-Allow-Reserved-Names': 'true',
'Host': 'localhost:80',
'User-Agent': 'test'
})], app._calls)
self.assertEqual(app.backend_user_agent, 'test')
self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests)
def test_delete_account(self):
account, container, obj = path_parts()
path = make_path_info(account)
client, app = get_client_app()
app.register('DELETE', path, swob.HTTPNoContent, {})
client.delete_account(account)
self.assertEqual(1, len(app._calls))
self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
def test_get_account_info(self):
class Response(object):
def __init__(self, containers, objects):
self.headers = {
'x-account-container-count': containers,
'x-account-object-count': objects,
}
self.status_int = 200
class InternalClient(internal_client.InternalClient):
def __init__(self, test, path, resp):
self.test = test
self.path = path
self.resp = resp
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
self.test.assertEqual('HEAD', method)
self.test.assertEqual(self.path, path)
self.test.assertEqual({}, headers)
self.test.assertEqual((2, 404), acceptable_statuses)
self.test.assertIsNone(body_file)
return self.resp
account, container, obj = path_parts()
path = make_path(account)
containers, objects = 10, 100
client = InternalClient(self, path, Response(containers, objects))
info = client.get_account_info(account)
self.assertEqual((containers, objects), info)
def test_get_account_info_404(self):
class Response(object):
def __init__(self):
self.headers = {
'x-account-container-count': 10,
'x-account-object-count': 100,
}
self.status_int = 404
class InternalClient(internal_client.InternalClient):
def __init__(self):
pass
def make_path(self, *a, **kw):
return 'some_path'
def make_request(self, *a, **kw):
return Response()
client = InternalClient()
info = client.get_account_info('some_account')
self.assertEqual((0, 0), info)
def test_get_account_metadata(self):
account, container, obj = path_parts()
path = make_path(account)
acceptable_statuses = 'some_status_list'
metadata_prefix = 'some_metadata_prefix'
client = GetMetadataInternalClient(
self, path, metadata_prefix, acceptable_statuses)
metadata = client.get_account_metadata(
account, metadata_prefix, acceptable_statuses)
self.assertEqual(client.metadata, metadata)
self.assertEqual(1, client.get_metadata_called)
def test_get_metadadata_with_acceptable_status(self):
account, container, obj = path_parts()
path = make_path_info(account)
client, app = get_client_app()
resp_headers = {'some-important-header': 'some value'}
app.register('GET', path, swob.HTTPOk, resp_headers)
metadata = client.get_account_metadata(
account, acceptable_statuses=(2, 4))
self.assertEqual(metadata['some-important-header'],
'some value')
app.register('GET', path, swob.HTTPNotFound, resp_headers)
metadata = client.get_account_metadata(
account, acceptable_statuses=(2, 4))
self.assertEqual(metadata['some-important-header'],
'some value')
app.register('GET', path, swob.HTTPServerError, resp_headers)
self.assertRaises(internal_client.UnexpectedResponse,
client.get_account_metadata, account,
acceptable_statuses=(2, 4))
def test_set_account_metadata(self):
account, container, obj = path_parts()
path = make_path_info(account)
client, app = get_client_app()
app.register('POST', path, swob.HTTPAccepted, {})
client.set_account_metadata(account, {'Color': 'Blue'},
metadata_prefix='X-Account-Meta-')
self.assertEqual([('POST', path, {
'X-Backend-Allow-Reserved-Names': 'true',
'Host': 'localhost:80',
'X-Account-Meta-Color': 'Blue',
'User-Agent': 'test'
})], app._calls)
self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
def test_set_account_metadata_plumbing(self):
account, container, obj = path_parts()
path = make_path(account)
metadata = 'some_metadata'
metadata_prefix = 'some_metadata_prefix'
acceptable_statuses = 'some_status_list'
client = SetMetadataInternalClient(
self, path, metadata, metadata_prefix, acceptable_statuses)
client.set_account_metadata(
account, metadata, metadata_prefix, acceptable_statuses)
self.assertEqual(1, client.set_metadata_called)
def test_container_exists(self):
class Response(object):
def __init__(self, status_int):
self.status_int = status_int
class InternalClient(internal_client.InternalClient):
def __init__(self, test, path, resp):
self.test = test
self.path = path
self.make_request_called = 0
self.resp = resp
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
self.make_request_called += 1
self.test.assertEqual('HEAD', method)
self.test.assertEqual(self.path, path)
self.test.assertEqual({}, headers)
self.test.assertEqual((2, 404), acceptable_statuses)
self.test.assertIsNone(body_file)
return self.resp
account, container, obj = path_parts()
path = make_path(account, container)
client = InternalClient(self, path, Response(200))
self.assertEqual(True, client.container_exists(account, container))
self.assertEqual(1, client.make_request_called)
client = InternalClient(self, path, Response(404))
self.assertEqual(False, client.container_exists(account, container))
self.assertEqual(1, client.make_request_called)
def test_create_container(self):
account, container, obj = path_parts()
path = make_path_info(account, container)
client, app = get_client_app()
app.register('PUT', path, swob.HTTPCreated, {})
client.create_container(account, container)
self.assertEqual([('PUT', path, {
'X-Backend-Allow-Reserved-Names': 'true',
'Host': 'localhost:80',
'User-Agent': 'test'
})], app._calls)
self.assertEqual(app.backend_user_agent, 'test')
self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests)
def test_create_container_plumbing(self):
class InternalClient(internal_client.InternalClient):
def __init__(self, test, path, headers):
self.test = test
self.path = path
self.headers = headers
self.make_request_called = 0
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
self.make_request_called += 1
self.test.assertEqual('PUT', method)
self.test.assertEqual(self.path, path)
self.test.assertEqual(self.headers, headers)
self.test.assertEqual((2,), acceptable_statuses)
self.test.assertIsNone(body_file)
account, container, obj = path_parts()
path = make_path(account, container)
headers = 'some_headers'
client = InternalClient(self, path, headers)
client.create_container(account, container, headers)
self.assertEqual(1, client.make_request_called)
def test_delete_container(self):
account, container, obj = path_parts()
path = make_path_info(account, container)
client, app = get_client_app()
app.register('DELETE', path, swob.HTTPNoContent, {})
client.delete_container(account, container)
self.assertEqual(1, len(app._calls))
self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
def test_delete_container_plumbing(self):
class InternalClient(internal_client.InternalClient):
def __init__(self, test, path):
self.test = test
self.path = path
self.make_request_called = 0
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
self.make_request_called += 1
self.test.assertEqual('DELETE', method)
self.test.assertEqual(self.path, path)
self.test.assertEqual({}, headers)
self.test.assertEqual((2, 404), acceptable_statuses)
self.test.assertIsNone(body_file)
account, container, obj = path_parts()
path = make_path(account, container)
client = InternalClient(self, path)
client.delete_container(account, container)
self.assertEqual(1, client.make_request_called)
def test_get_container_metadata(self):
account, container, obj = path_parts()
path = make_path(account, container)
metadata_prefix = 'some_metadata_prefix'
acceptable_statuses = 'some_status_list'
client = GetMetadataInternalClient(
self, path, metadata_prefix, acceptable_statuses)
metadata = client.get_container_metadata(
account, container, metadata_prefix, acceptable_statuses)
self.assertEqual(client.metadata, metadata)
self.assertEqual(1, client.get_metadata_called)
def test_iter_objects(self):
account, container, obj = path_parts()
path = make_path(account, container)
marker = 'some_maker'
end_marker = 'some_end_marker'
prefix = 'some_prefix'
acceptable_statuses = 'some_status_list'
items = '0 1 2'.split()
client = IterInternalClient(
self, path, marker, end_marker, prefix, acceptable_statuses, items)
ret_items = []
for obj in client.iter_objects(
account, container, marker, end_marker, prefix,
acceptable_statuses):
ret_items.append(obj)
self.assertEqual(items, ret_items)
def test_set_container_metadata(self):
account, container, obj = path_parts()
path = make_path_info(account, container)
client, app = get_client_app()
app.register('POST', path, swob.HTTPAccepted, {})
client.set_container_metadata(account, container, {'Color': 'Blue'},
metadata_prefix='X-Container-Meta-')
self.assertEqual([('POST', path, {
'X-Backend-Allow-Reserved-Names': 'true',
'Host': 'localhost:80',
'X-Container-Meta-Color': 'Blue',
'User-Agent': 'test'
})], app._calls)
self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
def test_set_container_metadata_plumbing(self):
account, container, obj = path_parts()
path = make_path(account, container)
metadata = 'some_metadata'
metadata_prefix = 'some_metadata_prefix'
acceptable_statuses = 'some_status_list'
client = SetMetadataInternalClient(
self, path, metadata, metadata_prefix, acceptable_statuses)
client.set_container_metadata(
account, container, metadata, metadata_prefix, acceptable_statuses)
self.assertEqual(1, client.set_metadata_called)
def test_delete_object(self):
account, container, obj = path_parts()
path = make_path_info(account, container, obj)
client, app = get_client_app()
app.register('DELETE', path, swob.HTTPNoContent, {})
client.delete_object(account, container, obj)
self.assertEqual(app.unclosed_requests, {})
self.assertEqual(1, len(app._calls))
self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
app.register('DELETE', path, swob.HTTPNotFound, {})
client.delete_object(account, container, obj)
self.assertEqual(app.unclosed_requests, {})
self.assertEqual(2, len(app._calls))
self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests)
def test_get_object_metadata(self):
account, container, obj = path_parts()
path = make_path(account, container, obj)
metadata_prefix = 'some_metadata_prefix'
acceptable_statuses = 'some_status_list'
client = GetMetadataInternalClient(
self, path, metadata_prefix, acceptable_statuses)
metadata = client.get_object_metadata(
account, container, obj, metadata_prefix,
acceptable_statuses)
self.assertEqual(client.metadata, metadata)
self.assertEqual(1, client.get_metadata_called)
def test_get_metadata_extra_headers(self):
class FakeApp(FakeSwift):
def __call__(self, env, start_response):
self.req_env = env
start_response('200 Ok', [('Content-Length', '0')])
return []
client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp())
headers = {'X-Foo': 'bar'}
client.get_object_metadata('account', 'container', 'obj',
headers=headers)
self.assertEqual(client.app.req_env['HTTP_X_FOO'], 'bar')
def test_get_object(self):
account, container, obj = path_parts()
path_info = make_path_info(account, container, obj)
client, app = get_client_app()
headers = {'foo': 'bar'}
body = b'some_object_body'
params = {'symlink': 'get'}
app.register('GET', path_info, swob.HTTPOk, headers, body)
req_headers = {'x-important-header': 'some_important_value'}
status_int, resp_headers, obj_iter = client.get_object(
account, container, obj, req_headers, params=params)
self.assertEqual(status_int // 100, 2)
for k, v in headers.items():
self.assertEqual(v, resp_headers[k])
self.assertEqual(b''.join(obj_iter), body)
self.assertEqual(resp_headers['content-length'], str(len(body)))
self.assertEqual(app.call_count, 1)
req_headers.update({
'host': 'localhost:80', # from swob.Request.blank
'x-backend-allow-reserved-names': 'true', # also from IC
'x-backend-storage-policy-index': '2', # from proxy-server app
'user-agent': 'test',
})
self.assertEqual(app.calls_with_headers, [(
'GET', path_info + '?symlink=get', HeaderKeyDict(req_headers))])
def test_iter_object_lines(self):
class FakeApp(FakeSwift):
def __init__(self, lines):
super(FakeApp, self).__init__()
self.lines = lines
def __call__(self, env, start_response):
start_response('200 Ok', [('Content-Length', '0')])
return [b'%s\n' % x for x in self.lines]
lines = b'line1 line2 line3'.split()
client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp(lines))
ret_lines = []
for line in client.iter_object_lines('account', 'container', 'object'):
ret_lines.append(line)
self.assertEqual(lines, ret_lines)
self.assertEqual(client.app._pipeline_final_app.backend_user_agent,
'some_agent')
def test_iter_object_lines_compressed_object(self):
class FakeApp(FakeSwift):
def __init__(self, lines):
super(FakeApp, self).__init__()
self.lines = lines
def __call__(self, env, start_response):
start_response('200 Ok', [('Content-Length', '0')])
return internal_client.CompressingFileReader(
BytesIO(b'\n'.join(self.lines)))
lines = b'line1 line2 line3'.split()
client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp(lines))
ret_lines = []
for line in client.iter_object_lines(
'account', 'container', 'object.gz'):
ret_lines.append(line)
self.assertEqual(lines, ret_lines)
def test_iter_object_lines_404(self):
class FakeApp(FakeSwift):
def __call__(self, env, start_response):
start_response('404 Not Found', [])
return [b'one\ntwo\nthree']
client = internal_client.InternalClient(
None, 'some_agent', 3, use_replication_network=False,
app=FakeApp())
lines = []
for line in client.iter_object_lines(
'some_account', 'some_container', 'some_object',
acceptable_statuses=(2, 404)):
lines.append(line)
self.assertEqual([], lines)
def test_set_object_metadata(self):
account, container, obj = path_parts()
path = make_path_info(account, container, obj)
client, app = get_client_app()
app.register('POST', path, swob.HTTPAccepted, {})
client.set_object_metadata(account, container, obj, {'Color': 'Blue'},
metadata_prefix='X-Object-Meta-')
self.assertEqual([('POST', path, {
'X-Backend-Allow-Reserved-Names': 'true',
'Host': 'localhost:80',
'X-Object-Meta-Color': 'Blue',
'User-Agent': 'test'
})], app._calls)
self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
def test_set_object_metadata_plumbing(self):
account, container, obj = path_parts()
path = make_path(account, container, obj)
metadata = 'some_metadata'
metadata_prefix = 'some_metadata_prefix'
acceptable_statuses = 'some_status_list'
client = SetMetadataInternalClient(
self, path, metadata, metadata_prefix, acceptable_statuses)
client.set_object_metadata(
account, container, obj, metadata, metadata_prefix,
acceptable_statuses)
self.assertEqual(1, client.set_metadata_called)
def test_upload_object(self):
account, container, obj = path_parts()
path = make_path_info(account, container, obj)
client, app = get_client_app()
app.register('PUT', path, swob.HTTPCreated, {})
client.upload_object(BytesIO(b'fobj'), account, container, obj)
self.assertEqual([('PUT', path, {
'Transfer-Encoding': 'chunked',
'X-Backend-Allow-Reserved-Names': 'true',
'Host': 'localhost:80',
'User-Agent': 'test'
})], app._calls)
self.assertEqual({}, app.unread_requests)
self.assertEqual({}, app.unclosed_requests)
self.assertEqual(app.backend_user_agent, 'test')
def test_upload_object_plumbing(self):
class InternalClient(internal_client.InternalClient):
def __init__(self, test, path, headers, fobj):
self.test = test
self.use_replication_network = False
self.path = path
self.headers = headers
self.fobj = fobj
self.make_request_called = 0
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None, params=None):
self.make_request_called += 1
self.test.assertEqual(self.path, path)
exp_headers = dict(self.headers)
exp_headers['Transfer-Encoding'] = 'chunked'
self.test.assertEqual(exp_headers, headers)
self.test.assertEqual(self.fobj, fobj)
fobj = 'some_fobj'
account, container, obj = path_parts()
path = make_path(account, container, obj)
headers = {'key': 'value'}
client = InternalClient(self, path, headers, fobj)
client.upload_object(fobj, account, container, obj, headers)
self.assertEqual(1, client.make_request_called)
def test_upload_object_not_chunked(self):
class InternalClient(internal_client.InternalClient):
def __init__(self, test, path, headers, fobj):
self.test = test
self.path = path
self.headers = headers
self.fobj = fobj
self.make_request_called = 0
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None, params=None):
self.make_request_called += 1
self.test.assertEqual(self.path, path)
exp_headers = dict(self.headers)
self.test.assertEqual(exp_headers, headers)
self.test.assertEqual(self.fobj, fobj)
fobj = 'some_fobj'
account, container, obj = path_parts()
path = make_path(account, container, obj)
headers = {'key': 'value', 'Content-Length': len(fobj)}
client = InternalClient(self, path, headers, fobj)
client.upload_object(fobj, account, container, obj, headers)
self.assertEqual(1, client.make_request_called)
class TestGetAuth(unittest.TestCase):
@mock.patch.object(urllib2, 'urlopen')
@mock.patch.object(urllib2, 'Request')
def test_ok(self, request, urlopen):
def getheader(name):
d = {'X-Storage-Url': 'url', 'X-Auth-Token': 'token'}
return d.get(name)
urlopen.return_value.info.return_value.getheader = getheader
url, token = internal_client.get_auth(
'http://127.0.0.1', 'user', 'key')
self.assertEqual(url, "url")
self.assertEqual(token, "token")
request.assert_called_with('http://127.0.0.1')
request.return_value.add_header.assert_any_call('X-Auth-User', 'user')
request.return_value.add_header.assert_any_call('X-Auth-Key', 'key')
def test_invalid_version(self):
self.assertRaises(SystemExit, internal_client.get_auth,
'http://127.0.0.1', 'user', 'key', auth_version=2.0)
class TestSimpleClient(unittest.TestCase):
def _test_get_head(self, request, urlopen, method):
mock_time_value = [1401224049.98]
def mock_time():
# global mock_time_value
mock_time_value[0] += 1
return mock_time_value[0]
with mock.patch('swift.common.internal_client.time', mock_time):
# basic request, only url as kwarg
request.return_value.get_type.return_value = "http"
urlopen.return_value.read.return_value = b''
urlopen.return_value.getcode.return_value = 200
urlopen.return_value.info.return_value = {'content-length': '345'}
sc = internal_client.SimpleClient(url='http://127.0.0.1')
logger = debug_logger('test-ic')
retval = sc.retry_request(
method, headers={'content-length': '123'}, logger=logger)
self.assertEqual(urlopen.call_count, 1)
request.assert_called_with('http://127.0.0.1?format=json',
headers={'content-length': '123'},
data=None)
self.assertEqual([{'content-length': '345'}, None], retval)
self.assertEqual(method, request.return_value.get_method())
self.assertEqual(logger.get_lines_for_level('debug'), [
'-> 2014-05-27T20:54:11 ' + method +
' http://127.0.0.1%3Fformat%3Djson 200 '
'123 345 1401224050.98 1401224051.98 1.0 -'
])
# Check if JSON is decoded
urlopen.return_value.read.return_value = b'{}'
retval = sc.retry_request(method)
self.assertEqual([{'content-length': '345'}, {}], retval)
# same as above, now with token
sc = internal_client.SimpleClient(url='http://127.0.0.1',
token='token')
retval = sc.retry_request(method)
request.assert_called_with('http://127.0.0.1?format=json',
headers={'X-Auth-Token': 'token'},
data=None)
self.assertEqual([{'content-length': '345'}, {}], retval)
# same as above, now with prefix
sc = internal_client.SimpleClient(url='http://127.0.0.1',
token='token')
retval = sc.retry_request(method, prefix="pre_")
request.assert_called_with(
'http://127.0.0.1?format=json&prefix=pre_',
headers={'X-Auth-Token': 'token'}, data=None)
self.assertEqual([{'content-length': '345'}, {}], retval)
# same as above, now with container name
retval = sc.retry_request(method, container='cont')
request.assert_called_with('http://127.0.0.1/cont?format=json',
headers={'X-Auth-Token': 'token'},
data=None)
self.assertEqual([{'content-length': '345'}, {}], retval)
# same as above, now with object name
retval = sc.retry_request(method, container='cont', name='obj')
request.assert_called_with('http://127.0.0.1/cont/obj',
headers={'X-Auth-Token': 'token'},
data=None)
self.assertEqual([{'content-length': '345'}, {}], retval)
@mock.patch.object(urllib2, 'urlopen')
@mock.patch.object(urllib2, 'Request')
def test_get(self, request, urlopen):
self._test_get_head(request, urlopen, 'GET')
@mock.patch.object(urllib2, 'urlopen')
@mock.patch.object(urllib2, 'Request')
def test_head(self, request, urlopen):
self._test_get_head(request, urlopen, 'HEAD')
@mock.patch.object(urllib2, 'urlopen')
@mock.patch.object(urllib2, 'Request')
def test_get_with_retries_all_failed(self, request, urlopen):
# Simulate a failing request, ensure retries done
request.return_value.get_type.return_value = "http"
urlopen.side_effect = urllib2.URLError('')
sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1)
with mock.patch('swift.common.internal_client.sleep') as mock_sleep:
self.assertRaises(urllib2.URLError, sc.retry_request, 'GET')
self.assertEqual(mock_sleep.call_count, 1)
self.assertEqual(request.call_count, 2)
self.assertEqual(urlopen.call_count, 2)
@mock.patch.object(urllib2, 'urlopen')
@mock.patch.object(urllib2, 'Request')
def test_get_with_retries(self, request, urlopen):
# First request fails, retry successful
request.return_value.get_type.return_value = "http"
mock_resp = mock.MagicMock()
mock_resp.read.return_value = b''
mock_resp.info.return_value = {}
urlopen.side_effect = [urllib2.URLError(''), mock_resp]
sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1,
token='token')
with mock.patch('swift.common.internal_client.sleep') as mock_sleep:
retval = sc.retry_request('GET')
self.assertEqual(mock_sleep.call_count, 1)
self.assertEqual(request.call_count, 2)
self.assertEqual(urlopen.call_count, 2)
request.assert_called_with('http://127.0.0.1?format=json', data=None,
headers={'X-Auth-Token': 'token'})
self.assertEqual([{}, None], retval)
self.assertEqual(sc.attempts, 2)
@mock.patch.object(urllib2, 'urlopen')
def test_get_with_retries_param(self, mock_urlopen):
mock_response = mock.MagicMock()
mock_response.read.return_value = b''
mock_response.info.return_value = {}
mock_urlopen.side_effect = internal_client.httplib.BadStatusLine('')
c = internal_client.SimpleClient(url='http://127.0.0.1', token='token')
self.assertEqual(c.retries, 5)
# first without retries param
with mock.patch('swift.common.internal_client.sleep') as mock_sleep:
self.assertRaises(internal_client.httplib.BadStatusLine,
c.retry_request, 'GET')
self.assertEqual(mock_sleep.call_count, 5)
self.assertEqual(mock_urlopen.call_count, 6)
# then with retries param
mock_urlopen.reset_mock()
with mock.patch('swift.common.internal_client.sleep') as mock_sleep:
self.assertRaises(internal_client.httplib.BadStatusLine,
c.retry_request, 'GET', retries=2)
self.assertEqual(mock_sleep.call_count, 2)
self.assertEqual(mock_urlopen.call_count, 3)
# and this time with a real response
mock_urlopen.reset_mock()
mock_urlopen.side_effect = [internal_client.httplib.BadStatusLine(''),
mock_response]
with mock.patch('swift.common.internal_client.sleep') as mock_sleep:
retval = c.retry_request('GET', retries=1)
self.assertEqual(mock_sleep.call_count, 1)
self.assertEqual(mock_urlopen.call_count, 2)
self.assertEqual([{}, None], retval)
@mock.patch.object(urllib2, 'urlopen')
def test_request_with_retries_with_HTTPError(self, mock_urlopen):
mock_response = mock.MagicMock()
mock_response.read.return_value = b''
c = internal_client.SimpleClient(url='http://127.0.0.1', token='token')
self.assertEqual(c.retries, 5)
for request_method in 'GET PUT POST DELETE HEAD COPY'.split():
mock_urlopen.reset_mock()
mock_urlopen.side_effect = urllib2.HTTPError(*[None] * 5)
with mock.patch('swift.common.internal_client.sleep') \
as mock_sleep:
self.assertRaises(exceptions.ClientException,
c.retry_request, request_method, retries=1)
self.assertEqual(mock_sleep.call_count, 1)
self.assertEqual(mock_urlopen.call_count, 2)
@mock.patch.object(urllib2, 'urlopen')
def test_request_container_with_retries_with_HTTPError(self,
mock_urlopen):
mock_response = mock.MagicMock()
mock_response.read.return_value = b''
c = internal_client.SimpleClient(url='http://127.0.0.1', token='token')
self.assertEqual(c.retries, 5)
for request_method in 'GET PUT POST DELETE HEAD COPY'.split():
mock_urlopen.reset_mock()
mock_urlopen.side_effect = urllib2.HTTPError(*[None] * 5)
with mock.patch('swift.common.internal_client.sleep') \
as mock_sleep:
self.assertRaises(exceptions.ClientException,
c.retry_request, request_method,
container='con', retries=1)
self.assertEqual(mock_sleep.call_count, 1)
self.assertEqual(mock_urlopen.call_count, 2)
@mock.patch.object(urllib2, 'urlopen')
def test_request_object_with_retries_with_HTTPError(self,
mock_urlopen):
mock_response = mock.MagicMock()
mock_response.read.return_value = b''
c = internal_client.SimpleClient(url='http://127.0.0.1', token='token')
self.assertEqual(c.retries, 5)
for request_method in 'GET PUT POST DELETE HEAD COPY'.split():
mock_urlopen.reset_mock()
mock_urlopen.side_effect = urllib2.HTTPError(*[None] * 5)
with mock.patch('swift.common.internal_client.sleep') \
as mock_sleep:
self.assertRaises(exceptions.ClientException,
c.retry_request, request_method,
container='con', name='obj', retries=1)
self.assertEqual(mock_sleep.call_count, 1)
self.assertEqual(mock_urlopen.call_count, 2)
@mock.patch.object(urllib2, 'urlopen')
def test_delete_object_with_404_no_retry(self, mock_urlopen):
mock_response = mock.MagicMock()
mock_response.read.return_value = b''
err_args = [None, 404, None, None, None]
mock_urlopen.side_effect = urllib2.HTTPError(*err_args)
with mock.patch('swift.common.internal_client.sleep') as mock_sleep, \
self.assertRaises(exceptions.ClientException) as caught:
internal_client.delete_object('http://127.0.0.1',
container='con', name='obj')
self.assertEqual(caught.exception.http_status, 404)
self.assertEqual(mock_sleep.call_count, 0)
self.assertEqual(mock_urlopen.call_count, 1)
@mock.patch.object(urllib2, 'urlopen')
def test_delete_object_with_409_no_retry(self, mock_urlopen):
mock_response = mock.MagicMock()
mock_response.read.return_value = b''
err_args = [None, 409, None, None, None]
mock_urlopen.side_effect = urllib2.HTTPError(*err_args)
with mock.patch('swift.common.internal_client.sleep') as mock_sleep, \
self.assertRaises(exceptions.ClientException) as caught:
internal_client.delete_object('http://127.0.0.1',
container='con', name='obj')
self.assertEqual(caught.exception.http_status, 409)
self.assertEqual(mock_sleep.call_count, 0)
self.assertEqual(mock_urlopen.call_count, 1)
def test_proxy(self):
# check that proxy arg is passed through to the urllib Request
scheme = 'http'
proxy_host = '127.0.0.1:80'
proxy = '%s://%s' % (scheme, proxy_host)
url = 'https://127.0.0.1:1/a'
mocked = 'swift.common.internal_client.urllib2.urlopen'
# module level methods
for func in (internal_client.put_object,
internal_client.delete_object):
with mock.patch(mocked) as mock_urlopen:
mock_urlopen.return_value = FakeConn()
func(url, container='c', name='o1', contents='', proxy=proxy,
timeout=0.1, retries=0)
self.assertEqual(1, mock_urlopen.call_count)
args, kwargs = mock_urlopen.call_args
self.assertEqual(1, len(args))
self.assertEqual(1, len(kwargs))
self.assertEqual(0.1, kwargs['timeout'])
self.assertTrue(isinstance(args[0], urllib2.Request))
self.assertEqual(proxy_host, args[0].host)
if six.PY2:
self.assertEqual(scheme, args[0].type)
else:
# TODO: figure out why this happens, whether py2 or py3 is
# messed up, whether we care, and what can be done about it
self.assertEqual('https', args[0].type)
# class methods
content = mock.MagicMock()
cl = internal_client.SimpleClient(url)
scenarios = ((cl.get_account, []),
(cl.get_container, ['c']),
(cl.put_container, ['c']),
(cl.put_object, ['c', 'o', content]))
for scenario in scenarios:
with mock.patch(mocked) as mock_urlopen:
mock_urlopen.return_value = FakeConn()
scenario[0](*scenario[1], proxy=proxy, timeout=0.1)
self.assertEqual(1, mock_urlopen.call_count)
args, kwargs = mock_urlopen.call_args
self.assertEqual(1, len(args))
self.assertEqual(1, len(kwargs))
self.assertEqual(0.1, kwargs['timeout'])
self.assertTrue(isinstance(args[0], urllib2.Request))
self.assertEqual(proxy_host, args[0].host)
if six.PY2:
self.assertEqual(scheme, args[0].type)
else:
# See above
self.assertEqual('https', args[0].type)
if __name__ == '__main__':
unittest.main()
View Git Blame Copy Permalink