openstack/python-swiftclient
Code Issues Proposed changes

Add additional headers for HEAD/GET/DELETE requests.

Browse Source 
Change-Id: I69276ba711057c122f97deac412e492e313c34dd
Closes-Bug: 1615830
This commit is contained in:
Charles Hsu 2016-09-19 23:18:18 +08:00
parent 0ec6b7b162
commit 6cf2bd6626
8 changed files with 426 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
swiftclient/client.py
View File

@ -696,7 +696,7 @@ def store_response(resp, response_dict):
def get_account(url, token, marker=None, limit=None, prefix=None,
end_marker=None, http_conn=None, full_listing=False,
service_token=None):
service_token=None, headers=None):
"""
Get a listing of containers for the account.
@ -711,20 +711,28 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:param service_token: service auth token
:param headers: additional headers to include in the request
:returns: a tuple of (response headers, a list of containers) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
req_headers = {'X-Auth-Token': token, 'Accept-Encoding': 'gzip'}
if service_token:
req_headers['X-Service-Token'] = service_token
if headers:
req_headers.update(headers)
if not http_conn:
http_conn = http_connection(url)
if full_listing:
rv = get_account(url, token, marker, limit, prefix,
end_marker, http_conn)
end_marker, http_conn, headers=req_headers)
listing = rv[1]
while listing:
marker = listing[-1]['name']
listing = get_account(url, token, marker, limit, prefix,
end_marker, http_conn)[1]
end_marker, http_conn,
headers=req_headers)[1]
if listing:
rv[1].extend(listing)
return rv
@ -739,14 +747,12 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
if end_marker:
qs += '&end_marker=%s' % quote(end_marker)
full_path = '%s?%s' % (parsed.path, qs)
headers = {'X-Auth-Token': token, 'Accept-Encoding': 'gzip'}
if service_token:
headers['X-Service-Token'] = service_token
method = 'GET'
conn.request(method, full_path, '', headers)
conn.request(method, full_path, '', req_headers)
resp = conn.getresponse()
body = resp.read()
http_log(("%s?%s" % (url, qs), method,), {'headers': headers}, resp, body)
http_log(("%s?%s" % (url, qs), method,), {'headers': req_headers},
resp, body)
resp_headers = resp_header_dict(resp)
if resp.status < 200 or resp.status >= 300:
@ -756,7 +762,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
return resp_headers, parse_api_response(resp_headers, body)
def head_account(url, token, http_conn=None, service_token=None):
def head_account(url, token, http_conn=None, headers=None,
service_token=None):
"""
Get account stats.
@ -764,6 +771,7 @@ def head_account(url, token, http_conn=None, service_token=None):
:param token: auth token
:param http_conn: a tuple of (parsed url, HTTPConnection object),
(If None, it will create the conn object)
:param headers: additional headers to include in the request
:param service_token: service auth token
:returns: a dict containing the response's headers (all header names will
be lowercase)
@ -774,13 +782,16 @@ def head_account(url, token, http_conn=None, service_token=None):
else:
parsed, conn = http_connection(url)
method = "HEAD"
headers = {'X-Auth-Token': token}
req_headers = {'X-Auth-Token': token}
if service_token:
headers['X-Service-Token'] = service_token
conn.request(method, parsed.path, '', headers)
req_headers['X-Service-Token'] = service_token
if headers:
req_headers.update(headers)
conn.request(method, parsed.path, '', req_headers)
resp = conn.getresponse()
body = resp.read()
http_log((url, method,), {'headers': headers}, resp, body)
http_log((url, method,), {'headers': req_headers}, resp, body)
if resp.status < 200 or resp.status >= 300:
raise ClientException.from_response(resp, 'Account HEAD failed', body)
resp_headers = resp_header_dict(resp)
@ -1047,7 +1058,7 @@ def post_container(url, token, container, headers, http_conn=None,
def delete_container(url, token, container, http_conn=None,
response_dict=None, service_token=None,
query_string=None):
query_string=None, headers=None):
"""
Delete a container
@ -1060,6 +1071,7 @@ def delete_container(url, token, container, http_conn=None,
the response - status, reason and headers
:param service_token: service auth token
:param query_string: if set will be appended with '?' to generated path
:param headers: additional headers to include in the request
:raises ClientException: HTTP DELETE request failed
"""
if http_conn:
@ -1067,7 +1079,12 @@ def delete_container(url, token, container, http_conn=None,
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
headers = {'X-Auth-Token': token}
if headers:
headers = dict(headers)
else:
headers = {}
headers['X-Auth-Token'] = token
if service_token:
headers['X-Service-Token'] = service_token
if query_string:
@ -1682,19 +1699,19 @@ class Connection(object):
if reset_func:
reset_func(func, *args, **kwargs)
def head_account(self):
def head_account(self, headers=None):
"""Wrapper for :func:`head_account`"""
return self._retry(None, head_account)
return self._retry(None, head_account, headers=headers)
def get_account(self, marker=None, limit=None, prefix=None,
end_marker=None, full_listing=False):
end_marker=None, full_listing=False, headers=None):
"""Wrapper for :func:`get_account`"""
# TODO(unknown): With full_listing=True this will restart the entire
# listing with each retry. Need to make a better version that just
# retries where it left off.
return self._retry(None, get_account, marker=marker, limit=limit,
prefix=prefix, end_marker=end_marker,
full_listing=full_listing)
full_listing=full_listing, headers=headers)
def post_account(self, headers, response_dict=None,
query_string=None, data=None):
@ -1733,11 +1750,12 @@ class Connection(object):
response_dict=response_dict)
def delete_container(self, container, response_dict=None,
query_string=None):
query_string=None, headers={}):
"""Wrapper for :func:`delete_container`"""
return self._retry(None, delete_container, container,
response_dict=response_dict,
query_string=query_string)
query_string=query_string,
headers=headers)
def head_object(self, container, obj, headers=None):
"""Wrapper for :func:`head_object`"""
@ -1808,11 +1826,12 @@ class Connection(object):
response_dict=response_dict)
def delete_object(self, container, obj, query_string=None,
response_dict=None):
response_dict=None, headers=None):
"""Wrapper for :func:`delete_object`"""
return self._retry(None, delete_object, container, obj,
query_string=query_string,
response_dict=response_dict)
response_dict=response_dict,
headers=headers)
def get_capabilities(self, url=None):
url = url or self.url

14
swiftclient/command_helpers.py
View File

@ -11,7 +11,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from swiftclient.utils import prt_bytes
from swiftclient.utils import prt_bytes, split_request_headers
POLICY_HEADER_PREFIX = 'x-account-storage-policy-'
@ -19,8 +19,9 @@ POLICY_HEADER_PREFIX = 'x-account-storage-policy-'
def stat_account(conn, options):
items = []
req_headers = split_request_headers(options.get('header', []))
headers = conn.head_account()
headers = conn.head_account(headers=req_headers)
if options['verbose'] > 1:
items.extend([
('StorageURL', conn.url),
@ -91,8 +92,11 @@ def print_account_stats(items, headers, output_manager):
def stat_container(conn, options, container):
headers = conn.head_container(container)
req_headers = split_request_headers(options.get('header', []))
headers = conn.head_container(container, headers=req_headers)
items = []
if options['verbose'] > 1:
path = '%s/%s' % (conn.url, container)
items.extend([
@ -137,7 +141,9 @@ def print_container_stats(items, headers, output_manager):
def stat_object(conn, options, container, obj):
headers = conn.head_object(container, obj)
req_headers = split_request_headers(options.get('header', []))
headers = conn.head_object(container, obj, headers=req_headers)
items = []
if options['verbose'] > 1:
path = '%s/%s/%s' % (conn.url, container, obj)

46
swiftclient/service.py
View File

@ -44,7 +44,7 @@ from swiftclient.command_helpers import (
)
from swiftclient.utils import (
config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG,
parse_api_response, report_traceback, n_groups
parse_api_response, report_traceback, n_groups, split_request_headers
)
from swiftclient.exceptions import ClientException
from swiftclient.multithreading import MultiThreadingManager
@ -279,15 +279,11 @@ def split_headers(options, prefix=''):
reporting.
"""
headers = {}
for item in options:
split_item = item.split(':', 1)
if len(split_item) == 2:
headers[(prefix + split_item[0]).title()] = split_item[1].strip()
else:
raise SwiftError(
"Metadata parameter %s must contain a ':'.\n%s"
% (item, "Example: 'Color:Blue' or 'Size:Large'")
)
try:
headers = split_request_headers(options, prefix)
except ValueError as e:
raise SwiftError(e)
return headers
@ -467,7 +463,8 @@ class SwiftService(object):
performed by this call::
{
'human': False
'human': False,
'header': []
}
:returns: Either a single dictionary containing stats about an account
@ -849,6 +846,7 @@ class SwiftService(object):
'long': False,
'prefix': None,
'delimiter': None,
'header': []
}
:returns: A generator for returning the results of the list operation
@ -884,10 +882,12 @@ class SwiftService(object):
def _list_account_job(conn, options, result_queue):
marker = ''
error = None
req_headers = split_headers(options.get('header', []))
try:
while True:
_, items = conn.get_account(
marker=marker, prefix=options['prefix']
marker=marker, prefix=options['prefix'],
headers=req_headers
)
if not items:
@ -943,11 +943,12 @@ class SwiftService(object):
def _list_container_job(conn, container, options, result_queue):
marker = options.get('marker', '')
error = None
req_headers = split_headers(options.get('header', []))
try:
while True:
_, items = conn.get_container(
container, marker=marker, prefix=options['prefix'],
delimiter=options['delimiter']
delimiter=options['delimiter'], headers=req_headers
)
if not items:
@ -2112,6 +2113,7 @@ class SwiftService(object):
'yes_all': False,
'leave_segments': False,
'prefix': None,
'header': [],
}
:returns: A generator for returning the results of the delete
@ -2250,6 +2252,8 @@ class SwiftService(object):
def _delete_object(self, conn, container, obj, options,
results_queue=None):
_headers = {}
_headers = split_headers(options.get('header', []))
res = {
'action': 'delete_object',
'container': container,
@ -2261,7 +2265,8 @@ class SwiftService(object):
if not options['leave_segments']:
try:
headers = conn.head_object(container, obj)
headers = conn.head_object(container, obj,
headers=_headers)
old_manifest = headers.get('x-object-manifest')
if config_true_value(headers.get('x-static-large-object')):
query_string = 'multipart-manifest=delete'
@ -2270,7 +2275,9 @@ class SwiftService(object):
raise
results_dict = {}
conn.delete_object(container, obj, query_string=query_string,
conn.delete_object(container, obj,
headers=_headers,
query_string=query_string,
response_dict=results_dict)
if old_manifest:
@ -2322,10 +2329,13 @@ class SwiftService(object):
return res
@staticmethod
def _delete_empty_container(conn, container):
def _delete_empty_container(conn, container, options):
results_dict = {}
_headers = {}
_headers = split_headers(options.get('header', []))
try:
conn.delete_container(container, response_dict=results_dict)
conn.delete_container(container, headers=_headers,
response_dict=results_dict)
res = {'success': True}
except Exception as err:
traceback, err_time = report_traceback()
@ -2363,7 +2373,7 @@ class SwiftService(object):
return
con_del = self.thread_manager.container_pool.submit(
self._delete_empty_container, container
self._delete_empty_container, container, options
)
con_del_res = get_future_result(con_del)

27
swiftclient/shell.py
View File

@ -58,6 +58,7 @@ def immediate_exit(signum, frame):
st_delete_options = '''[--all] [--leave-segments]
[--object-threads <threads>]
[--container-threads <threads>]
[--header <header:value>]
[<container> [<object>] [...]]
'''
@ -72,6 +73,9 @@ Positional arguments:
Optional arguments:
-a, --all Delete all containers and objects.
--leave-segments Do not delete segments of manifest objects.
-H, --header <header:value>
Adds a custom request header to use for deleting
objects or an entire container .
--object-threads <threads>
Number of threads to use for deleting objects.
Default is 10.
@ -88,6 +92,11 @@ def st_delete(parser, args, output_manager):
parser.add_argument(
'-p', '--prefix', dest='prefix',
help='Only delete items beginning with the <prefix>.')
parser.add_argument(
'-H', '--header', action='append', dest='header',
default=[],
help='Adds a custom request header to use for deleting objects '
'or an entire container.')
parser.add_argument(
'--leave-segments', action='store_true',
dest='leave_segments', default=False,
@ -445,7 +454,8 @@ def st_download(parser, args, output_manager):
st_list_options = '''[--long] [--lh] [--totals] [--prefix <prefix>]
[--delimiter <delimiter>] [<container>]
[--delimiter <delimiter>] [--header <header:value>]
[<container>]
'''
st_list_help = '''
@ -465,6 +475,8 @@ Optional arguments:
Roll up items with the given delimiter. For containers
only. See OpenStack Swift API documentation for what
this means.
-H, --header <header:value>
Adds a custom request header to use for listing.
'''.strip('\n')
@ -541,6 +553,10 @@ def st_list(parser, args, output_manager):
help='Roll up items with the given delimiter. For containers '
'only. See OpenStack Swift API documentation for '
'what this means.')
parser.add_argument(
'-H', '--header', action='append', dest='header',
default=[],
help='Adds a custom request header to use for listing.')
options, args = parse_args(parser, args)
args = args[1:]
if options['delimiter'] and not args:
@ -580,7 +596,7 @@ def st_list(parser, args, output_manager):
output_manager.error(e.value)
st_stat_options = '''[--lh]
st_stat_options = '''[--lh] [--header <header:value>]
[<container> [<object>]]
'''
@ -594,6 +610,8 @@ Positional arguments:
Optional arguments:
--lh Report sizes in human readable format similar to
ls -lh.
-H, --header <header:value>
Adds a custom request header to use for stat.
'''.strip('\n')
@ -601,6 +619,11 @@ def st_stat(parser, args, output_manager):
parser.add_argument(
'--lh', dest='human', action='store_true', default=False,
help='Report sizes in human readable format similar to ls -lh.')
parser.add_argument(
'-H', '--header', action='append', dest='header',
default=[],
help='Adds a custom request header to use for stat.')
options, args = parse_args(parser, args)
args = args[1:]

14
swiftclient/utils.py
View File

@ -146,6 +146,20 @@ def parse_api_response(headers, body):
return json.loads(body.decode(charset))
def split_request_headers(options, prefix=''):
headers = {}
for item in options:
split_item = item.split(':', 1)
if len(split_item) == 2:
headers[(prefix + split_item[0]).title()] = split_item[1].strip()
else:
raise ValueError(
"Metadata parameter %s must contain a ':'.\n%s"
% (item, "Example: 'Color:Blue' or 'Size:Large'")
)
return headers
def report_traceback():
"""
Reports a timestamp and full traceback for a given exception.

158
tests/unit/test_service.py
View File

@ -291,9 +291,33 @@ class TestServiceDelete(_TestServiceBase):
s = SwiftService()
r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q)
mock_conn.head_object.assert_called_once_with('test_c', 'test_o')
mock_conn.head_object.assert_called_once_with('test_c', 'test_o',
headers={})
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o', query_string=None, response_dict={}
'test_c', 'test_o', query_string=None, response_dict={},
headers={}
)
self.assertEqual(expected_r, r)
def test_delete_object_with_headers(self):
mock_q = Queue()
mock_conn = self._get_mock_connection()
mock_conn.head_object = Mock(return_value={})
expected_r = self._get_expected({
'action': 'delete_object',
'success': True
})
opt_c = self.opts.copy()
opt_c['header'] = ['Skip-Middleware: Test']
s = SwiftService()
r = s._delete_object(mock_conn, 'test_c', 'test_o', opt_c, mock_q)
mock_conn.head_object.assert_called_once_with(
'test_c', 'test_o', headers={'Skip-Middleware': 'Test'})
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o', query_string=None, response_dict={},
headers={'Skip-Middleware': 'Test'}
)
self.assertEqual(expected_r, r)
@ -317,9 +341,11 @@ class TestServiceDelete(_TestServiceBase):
r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q)
after = time.time()
mock_conn.head_object.assert_called_once_with('test_c', 'test_o')
mock_conn.head_object.assert_called_once_with('test_c', 'test_o',
headers={})
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o', query_string=None, response_dict={}
'test_c', 'test_o', query_string=None, response_dict={},
headers={}
)
self.assertEqual(expected_r, r)
self.assertGreaterEqual(r['error_timestamp'], before)
@ -342,11 +368,13 @@ class TestServiceDelete(_TestServiceBase):
s = SwiftService()
r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q)
mock_conn.head_object.assert_called_once_with('test_c', 'test_o')
mock_conn.head_object.assert_called_once_with('test_c', 'test_o',
headers={})
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o',
query_string='multipart-manifest=delete',
response_dict={}
response_dict={},
headers={}
)
self.assertEqual(expected_r, r)
@ -381,7 +409,8 @@ class TestServiceDelete(_TestServiceBase):
self.assertEqual(expected_r, r)
expected = [
mock.call('test_c', 'test_o', query_string=None, response_dict={}),
mock.call('test_c', 'test_o', query_string=None, response_dict={},
headers={}),
mock.call('manifest_c', 'test_seg_1', response_dict={}),
mock.call('manifest_c', 'test_seg_2', response_dict={})]
mock_conn.delete_object.assert_has_calls(expected, any_order=True)
@ -394,10 +423,28 @@ class TestServiceDelete(_TestServiceBase):
'object': None
})
r = SwiftService._delete_empty_container(mock_conn, 'test_c')
r = SwiftService._delete_empty_container(mock_conn, 'test_c',
self.opts)
mock_conn.delete_container.assert_called_once_with(
'test_c', response_dict={}
'test_c', response_dict={}, headers={}
)
self.assertEqual(expected_r, r)
def test_delete_empty_container_with_headers(self):
mock_conn = self._get_mock_connection()
expected_r = self._get_expected({
'action': 'delete_container',
'success': True,
'object': None
})
opt_c = self.opts.copy()
opt_c['header'] = ['Skip-Middleware: Test']
r = SwiftService._delete_empty_container(mock_conn, 'test_c', opt_c)
mock_conn.delete_container.assert_called_once_with(
'test_c', response_dict={}, headers={'Skip-Middleware': 'Test'}
)
self.assertEqual(expected_r, r)
@ -415,11 +462,11 @@ class TestServiceDelete(_TestServiceBase):
before = time.time()
s = SwiftService()
r = s._delete_empty_container(mock_conn, 'test_c')
r = s._delete_empty_container(mock_conn, 'test_c', {})
after = time.time()
mock_conn.delete_container.assert_called_once_with(
'test_c', response_dict={}
'test_c', response_dict={}, headers={}
)
self.assertEqual(expected_r, r)
self.assertGreaterEqual(r['error_timestamp'], before)
@ -665,6 +712,34 @@ class TestServiceList(_TestServiceBase):
self.assertEqual(expected_r_long, self._get_queue(mock_q))
self.assertIsNone(self._get_queue(mock_q))
def test_list_account_with_headers(self):
mock_q = Queue()
mock_conn = self._get_mock_connection()
get_account_returns = [
(None, [{'name': 'test_c'}]),
(None, [])
]
mock_conn.get_account = Mock(side_effect=get_account_returns)
expected_r = self._get_expected({
'action': 'list_account_part',
'success': True,
'listing': [{'name': 'test_c'}],
'marker': ''
})
opt_c = self.opts.copy()
opt_c['header'] = ['Skip-Middleware: True']
SwiftService._list_account_job(
mock_conn, opt_c, mock_q
)
self.assertEqual(expected_r, self._get_queue(mock_q))
self.assertIsNone(self._get_queue(mock_q))
self.assertEqual(mock_conn.get_account.mock_calls, [
mock.call(headers={'Skip-Middleware': 'True'}, marker='',
prefix=None),
mock.call(headers={'Skip-Middleware': 'True'}, marker='test_c',
prefix=None)])
def test_list_account_exception(self):
mock_q = Queue()
mock_conn = self._get_mock_connection()
@ -682,7 +757,7 @@ class TestServiceList(_TestServiceBase):
mock_conn, self.opts, mock_q)
mock_conn.get_account.assert_called_once_with(
marker='', prefix=None
marker='', prefix=None, headers={}
)
self.assertEqual(expected_r, self._get_queue(mock_q))
self.assertIsNone(self._get_queue(mock_q))
@ -767,6 +842,37 @@ class TestServiceList(_TestServiceBase):
self.assertIsNone(self._get_queue(mock_q))
def test_list_container_with_headers(self):
mock_q = Queue()
mock_conn = self._get_mock_connection()
get_container_returns = [
(None, [{'name': 'test_o'}]),
(None, [])
]
mock_conn.get_container = Mock(side_effect=get_container_returns)
expected_r = self._get_expected({
'action': 'list_container_part',
'container': 'test_c',
'success': True,
'listing': [{'name': 'test_o'}],
'marker': ''
})
opt_c = self.opts.copy()
opt_c['header'] = ['Skip-Middleware: Test']
SwiftService._list_container_job(
mock_conn, 'test_c', opt_c, mock_q
)
self.assertEqual(expected_r, self._get_queue(mock_q))
self.assertIsNone(self._get_queue(mock_q))
self.assertEqual(mock_conn.get_container.mock_calls, [
mock.call('test_c', headers={'Skip-Middleware': 'Test'},
delimiter='', marker='', prefix=None),
mock.call('test_c', headers={'Skip-Middleware': 'Test'},
delimiter='', marker='test_o', prefix=None)])
def test_list_container_exception(self):
mock_q = Queue()
mock_conn = self._get_mock_connection()
@ -786,7 +892,7 @@ class TestServiceList(_TestServiceBase):
)
mock_conn.get_container.assert_called_once_with(
'test_c', marker='', delimiter='', prefix=None
'test_c', marker='', delimiter='', prefix=None, headers={}
)
self.assertEqual(expected_r, self._get_queue(mock_q))
self.assertIsNone(self._get_queue(mock_q))
@ -1397,11 +1503,13 @@ class TestServiceUpload(_TestServiceBase):
mock_conn.head_object.assert_called_with('test_c', 'test_o')
expected = [
mock.call('test_c_segments', prefix='test_o/prefix',
marker='', delimiter=None),
marker='', delimiter=None, headers={}),
mock.call('test_c_segments', prefix='test_o/prefix',
marker="test_o/prefix/01", delimiter=None),
marker="test_o/prefix/01", delimiter=None,
headers={}),
mock.call('test_c_segments', prefix='test_o/prefix',
marker="test_o/prefix/02", delimiter=None),
marker="test_o/prefix/02", delimiter=None,
headers={}),
]
mock_conn.get_container.assert_has_calls(expected)
@ -2102,15 +2210,18 @@ class TestServiceDownload(_TestServiceBase):
mock.call('test_c_segments',
delimiter=None,
prefix='test_o/prefix',
marker=''),
marker='',
headers={}),
mock.call('test_c_segments',
delimiter=None,
prefix='test_o/prefix',
marker='test_o/prefix/2'),
marker='test_o/prefix/2',
headers={}),
mock.call('test_c_segments',
delimiter=None,
prefix='test_o/prefix',
marker='test_o/prefix/3')])
marker='test_o/prefix/3',
headers={})])
def test_download_object_job_skip_identical_nested_slo(self):
with tempfile.NamedTemporaryFile() as f:
@ -2243,15 +2354,18 @@ class TestServiceDownload(_TestServiceBase):
mock.call('test_c_segments',
delimiter=None,
prefix='test_o/prefix',
marker=''),
marker='',
headers={}),
mock.call('test_c_segments',
delimiter=None,
prefix='test_o/prefix',
marker='test_o/prefix/2'),
marker='test_o/prefix/2',
headers={}),
mock.call('test_c_segments',
delimiter=None,
prefix='test_o/prefix',
marker='test_o/prefix/3')])
marker='test_o/prefix/3',
headers={})])
self.assertEqual(mock_conn.get_object.mock_calls, [
mock.call('test_c',
'test_o',

197
tests/unit/test_shell.py
View File

@ -142,6 +142,28 @@ class TestShell(unittest.TestCase):
' Objects: 2\n'
' Bytes: 3\n')
@mock.patch('swiftclient.service.Connection')
def test_stat_account_with_headers(self, connection):
argv = ["", "stat", "-H", "Skip-Middleware: Test"]
return_headers = {
'x-account-container-count': '1',
'x-account-object-count': '2',
'x-account-bytes-used': '3',
'content-length': 0,
'date': ''}
connection.return_value.head_account.return_value = return_headers
connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account'
with CaptureOutput() as output:
swiftclient.shell.main(argv)
self.assertEqual(output.out,
' Account: AUTH_account\n'
'Containers: 1\n'
' Objects: 2\n'
' Bytes: 3\n')
self.assertEqual(connection.return_value.head_account.mock_calls, [
mock.call(headers={'Skip-Middleware': 'Test'})])
@mock.patch('swiftclient.service.Connection')
def test_stat_container(self, connection):
return_headers = {
@ -168,6 +190,34 @@ class TestShell(unittest.TestCase):
' Sync To: other\n'
' Sync Key: secret\n')
@mock.patch('swiftclient.service.Connection')
def test_stat_container_with_headers(self, connection):
return_headers = {
'x-container-object-count': '1',
'x-container-bytes-used': '2',
'x-container-read': 'test2:tester2',
'x-container-write': 'test3:tester3',
'x-container-sync-to': 'other',
'x-container-sync-key': 'secret',
}
argv = ["", "stat", "container", "-H", "Skip-Middleware: Test"]
connection.return_value.head_container.return_value = return_headers
connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account'
with CaptureOutput() as output:
swiftclient.shell.main(argv)
self.assertEqual(output.out,
' Account: AUTH_account\n'
'Container: container\n'
' Objects: 1\n'
' Bytes: 2\n'
' Read ACL: test2:tester2\n'
'Write ACL: test3:tester3\n'
' Sync To: other\n'
' Sync Key: secret\n')
self.assertEqual(connection.return_value.head_container.mock_calls, [
mock.call('container', headers={'Skip-Middleware': 'Test'})])
@mock.patch('swiftclient.service.Connection')
def test_stat_object(self, connection):
return_headers = {
@ -194,6 +244,36 @@ class TestShell(unittest.TestCase):
' ETag: md5\n'
' Manifest: manifest\n')
@mock.patch('swiftclient.service.Connection')
def test_stat_object_with_headers(self, connection):
return_headers = {
'x-object-manifest': 'manifest',
'etag': 'md5',
'last-modified': 'yesterday',
'content-type': 'text/plain',
'content-length': 42,
}
argv = ["", "stat", "container", "object",
"-H", "Skip-Middleware: Test"]
connection.return_value.head_object.return_value = return_headers
connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account'
with CaptureOutput() as output:
swiftclient.shell.main(argv)
self.assertEqual(output.out,
' Account: AUTH_account\n'
' Container: container\n'
' Object: object\n'
' Content Type: text/plain\n'
'Content Length: 42\n'
' Last Modified: yesterday\n'
' ETag: md5\n'
' Manifest: manifest\n')
self.assertEqual(connection.return_value.head_object.mock_calls, [
mock.call('container', 'object',
headers={'Skip-Middleware': 'Test'})])
@mock.patch('swiftclient.service.Connection')
def test_list_account(self, connection):
# Test account listing
@ -206,8 +286,28 @@ class TestShell(unittest.TestCase):
with CaptureOutput() as output:
swiftclient.shell.main(argv)
calls = [mock.call(marker='', prefix=None),
mock.call(marker='container', prefix=None)]
calls = [mock.call(marker='', prefix=None, headers={}),
mock.call(marker='container', prefix=None, headers={})]
connection.return_value.get_account.assert_has_calls(calls)
self.assertEqual(output.out, 'container\n')
@mock.patch('swiftclient.service.Connection')
def test_list_account_with_headers(self, connection):
# Test account listing
connection.return_value.get_account.side_effect = [
[None, [{'name': 'container'}]],
[None, []],
]
argv = ["", "list", '-H', 'Skip-Custom-Middleware: True']
with CaptureOutput() as output:
swiftclient.shell.main(argv)
calls = [mock.call(marker='', prefix=None,
headers={'Skip-Custom-Middleware': 'True'}),
mock.call(marker='container', prefix=None,
headers={'Skip-Custom-Middleware': 'True'})]
connection.return_value.get_account.assert_has_calls(calls)
self.assertEqual(output.out, 'container\n')
@ -223,8 +323,8 @@ class TestShell(unittest.TestCase):
argv = ["", "list", "--lh"]
with CaptureOutput() as output:
swiftclient.shell.main(argv)
calls = [mock.call(marker='', prefix=None),
mock.call(marker='container', prefix=None)]
calls = [mock.call(marker='', prefix=None, headers={}),
mock.call(marker='container', prefix=None, headers={})]
connection.return_value.get_account.assert_has_calls(calls)
self.assertEqual(output.out,
@ -243,8 +343,8 @@ class TestShell(unittest.TestCase):
argv = ["", "list", "--lh"]
with CaptureOutput() as output:
swiftclient.shell.main(argv)
calls = [mock.call(marker='', prefix=None),
mock.call(marker='container', prefix=None)]
calls = [mock.call(marker='', prefix=None, headers={}),
mock.call(marker='container', prefix=None, headers={})]
connection.return_value.get_account.assert_has_calls(calls)
self.assertEqual(output.out,
@ -273,7 +373,7 @@ class TestShell(unittest.TestCase):
argv = ["", "list", "--lh", "--totals"]
with CaptureOutput() as output:
swiftclient.shell.main(argv)
calls = [mock.call(marker='', prefix=None)]
calls = [mock.call(marker='', prefix=None, headers={})]
connection.return_value.get_account.assert_has_calls(calls)
self.assertEqual(output.out, ' 6 3\n')
@ -287,9 +387,10 @@ class TestShell(unittest.TestCase):
with CaptureOutput() as output:
swiftclient.shell.main(argv)
calls = [
mock.call('container', marker='', delimiter=None, prefix=None),
mock.call('container', marker='',
delimiter=None, prefix=None, headers={}),
mock.call('container', marker='object_a',
delimiter=None, prefix=None)]
delimiter=None, prefix=None, headers={})]
connection.return_value.get_container.assert_has_calls(calls)
self.assertEqual(output.out, 'object_a\n')
@ -305,9 +406,10 @@ class TestShell(unittest.TestCase):
with CaptureOutput() as output:
swiftclient.shell.main(argv)
calls = [
mock.call('container', marker='', delimiter=None, prefix=None),
mock.call('container', marker='',
delimiter=None, prefix=None, headers={}),
mock.call('container', marker='object_a',
delimiter=None, prefix=None)]
delimiter=None, prefix=None, headers={})]
connection.return_value.get_container.assert_has_calls(calls)
self.assertEqual(output.out,
@ -315,6 +417,26 @@ class TestShell(unittest.TestCase):
' type/content object_a\n'
' 0\n')
@mock.patch('swiftclient.service.Connection')
def test_list_container_with_headers(self, connection):
connection.return_value.get_container.side_effect = [
[None, [{'name': 'object_a'}]],
[None, []],
]
argv = ["", "list", "container", "-H", "Skip-Middleware: Test"]
with CaptureOutput() as output:
swiftclient.shell.main(argv)
calls = [
mock.call('container', marker='',
delimiter=None, prefix=None,
headers={'Skip-Middleware': 'Test'}),
mock.call('container', marker='object_a',
delimiter=None, prefix=None,
headers={'Skip-Middleware': 'Test'})]
connection.return_value.get_container.assert_has_calls(calls)
self.assertEqual(output.out, 'object_a\n')
@mock.patch('swiftclient.service.makedirs')
@mock.patch('swiftclient.service.Connection')
def test_download(self, connection, makedirs):
@ -835,19 +957,19 @@ class TestShell(unittest.TestCase):
swiftclient.shell.main(argv)
connection.return_value.delete_object.assert_has_calls([
mock.call('container', 'object', query_string=None,
response_dict={}),
response_dict={}, headers={}),
mock.call('container', 'obj\xe9ct2', query_string=None,
response_dict={}),
response_dict={}, headers={}),
mock.call('container2', 'object', query_string=None,
response_dict={})], any_order=True)
response_dict={}, headers={})], any_order=True)
self.assertEqual(3, connection.return_value.delete_object.call_count,
'Expected 3 calls but found\n%r'
% connection.return_value.delete_object.mock_calls)
self.assertEqual(
connection.return_value.delete_container.mock_calls, [
mock.call('container', response_dict={}),
mock.call('container2', response_dict={}),
mock.call('empty_container', response_dict={})])
mock.call('container', response_dict={}, headers={}),
mock.call('container2', response_dict={}, headers={}),
mock.call('empty_container', response_dict={}, headers={})])
@mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
lambda *a: True)
@ -900,9 +1022,9 @@ class TestShell(unittest.TestCase):
connection.return_value.post_account.mock_calls[2])
self.assertEqual(
connection.return_value.delete_container.mock_calls, [
mock.call('container', response_dict={}),
mock.call('container2', response_dict={}),
mock.call('empty_container', response_dict={})])
mock.call('container', response_dict={}, headers={}),
mock.call('container2', response_dict={}, headers={}),
mock.call('empty_container', response_dict={}, headers={})])
@mock.patch('swiftclient.service.Connection')
def test_delete_bulk_account_with_capabilities(self, connection):
@ -957,9 +1079,9 @@ class TestShell(unittest.TestCase):
response_dict={})])
self.assertEqual(
connection.return_value.delete_container.mock_calls, [
mock.call('container', response_dict={}),
mock.call('container2', response_dict={}),
mock.call('empty_container', response_dict={})])
mock.call('container', response_dict={}, headers={}),
mock.call('container2', response_dict={}, headers={}),
mock.call('empty_container', response_dict={}, headers={})])
self.assertEqual(connection.return_value.get_capabilities.mock_calls,
[mock.call(None)]) # only one /info request
@ -976,9 +1098,29 @@ class TestShell(unittest.TestCase):
connection.return_value.head_object.return_value = {}
swiftclient.shell.main(argv)
connection.return_value.delete_container.assert_called_with(
'container', response_dict={})
'container', response_dict={}, headers={})
connection.return_value.delete_object.assert_called_with(
'container', 'object', query_string=None, response_dict={})
'container', 'object', query_string=None, response_dict={},
headers={})
@mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
lambda *a: False)
@mock.patch('swiftclient.service.Connection')
def test_delete_container_headers(self, connection):
connection.return_value.get_container.side_effect = [
[None, [{'name': 'object'}]],
[None, []],
]
connection.return_value.attempts = 0
argv = ["", "delete", "container", "-H", "Skip-Middleware: Test"]
connection.return_value.head_object.return_value = {}
swiftclient.shell.main(argv)
connection.return_value.delete_container.assert_called_with(
'container', response_dict={},
headers={'Skip-Middleware': 'Test'})
connection.return_value.delete_object.assert_called_with(
'container', 'object', query_string=None, response_dict={},
headers={'Skip-Middleware': 'Test'})
@mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
lambda *a: True)
@ -1000,7 +1142,7 @@ class TestShell(unittest.TestCase):
'Accept': 'application/json'},
response_dict={})
connection.return_value.delete_container.assert_called_with(
'container', response_dict={})
'container', response_dict={}, headers={})
def test_delete_verbose_output_utf8(self):
container = 't\u00e9st_c'
@ -1043,7 +1185,8 @@ class TestShell(unittest.TestCase):
connection.return_value.attempts = 0
swiftclient.shell.main(argv)
connection.return_value.delete_object.assert_called_with(
'container', 'object', query_string=None, response_dict={})
'container', 'object', query_string=None, response_dict={},
headers={})
@mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
lambda *a: True)

2
tests/unit/test_swiftclient.py
View File

@ -3003,7 +3003,7 @@ class TestServiceToken(MockHttpTest):
with mock.patch('swiftclient.client.http_connection',
self.fake_http_connection(202)):
conn = self.get_connection()
conn.delete_object('container1', 'obj1', 'a_string')
conn.delete_object('container1', 'obj1', query_string='a_string')
self.assertEqual(1, len(self.request_log), self.request_log)
for actual in self.iter_request_log():
self.assertEqual('DELETE', actual['method'])