object versioning features

* add --versions to list
 * add --versions to delete
 * add --version-id to stat
 * add --version-id to delete
 * add --version-id to download

Change-Id: I89802064921778fee7efe57c7d60c976cdde3a27
This commit is contained in:
Clay Gerrard 2019-10-29 09:59:03 -05:00 committed by Tim Burke
parent 02e8f4f228
commit 78edffa46c
6 changed files with 611 additions and 59 deletions

View File

@ -921,7 +921,7 @@ def post_account(url, token, headers, http_conn=None, response_dict=None,
def get_container(url, token, container, marker=None, limit=None,
prefix=None, delimiter=None, end_marker=None,
path=None, http_conn=None,
version_marker=None, path=None, http_conn=None,
full_listing=False, service_token=None, headers=None,
query_string=None):
"""
@ -935,6 +935,7 @@ def get_container(url, token, container, marker=None, limit=None,
:param prefix: prefix query
:param delimiter: string to delimit the queries on
:param end_marker: marker query
:param version_marker: version marker query
:param path: path query (equivalent: "delimiter=/" and "prefix=path/")
:param http_conn: a tuple of (parsed url, HTTPConnection object),
(If None, it will create the conn object)
@ -951,17 +952,20 @@ def get_container(url, token, container, marker=None, limit=None,
http_conn = http_connection(url)
if full_listing:
rv = get_container(url, token, container, marker, limit, prefix,
delimiter, end_marker, path, http_conn,
service_token=service_token, headers=headers)
delimiter, end_marker, version_marker, path=path,
http_conn=http_conn, service_token=service_token,
headers=headers)
listing = rv[1]
while listing:
if not delimiter:
marker = listing[-1]['name']
else:
marker = listing[-1].get('name', listing[-1].get('subdir'))
version_marker = listing[-1].get('version_id')
listing = get_container(url, token, container, marker, limit,
prefix, delimiter, end_marker, path,
http_conn, service_token=service_token,
prefix, delimiter, end_marker,
version_marker, path, http_conn,
service_token=service_token,
headers=headers)[1]
if listing:
rv[1].extend(listing)
@ -979,6 +983,8 @@ def get_container(url, token, container, marker=None, limit=None,
qs += '&delimiter=%s' % quote(delimiter)
if end_marker:
qs += '&end_marker=%s' % quote(end_marker)
if version_marker:
qs += '&version_marker=%s' % quote(version_marker)
if path:
qs += '&path=%s' % quote(path)
if query_string:
@ -1816,15 +1822,17 @@ class Connection(object):
return self._retry(None, head_container, container, headers=headers)
def get_container(self, container, marker=None, limit=None, prefix=None,
delimiter=None, end_marker=None, path=None,
full_listing=False, headers=None, query_string=None):
delimiter=None, end_marker=None, version_marker=None,
path=None, full_listing=False, headers=None,
query_string=None):
"""Wrapper for :func:`get_container`"""
# TODO(unknown): With full_listing=True this will restart the entire
# listing with each retry. Need to make a better version that just
# retries where it left off.
return self._retry(None, get_container, container, marker=marker,
limit=limit, prefix=prefix, delimiter=delimiter,
end_marker=end_marker, path=path,
end_marker=end_marker,
version_marker=version_marker, path=path,
full_listing=full_listing, headers=headers,
query_string=query_string)

View File

@ -143,7 +143,11 @@ def print_container_stats(items, headers, output_manager):
def stat_object(conn, options, container, obj):
req_headers = split_request_headers(options.get('header', []))
headers = conn.head_object(container, obj, headers=req_headers)
query_string = None
if options.get('version_id') is not None:
query_string = 'version-id=%s' % options['version_id']
headers = conn.head_object(container, obj, headers=req_headers,
query_string=query_string)
items = []
if options['verbose'] > 1:
path = '%s/%s/%s' % (conn.url, container, obj)

View File

@ -86,6 +86,9 @@ class SwiftError(Exception):
value += " segment:%s" % self.segment
return value
def __repr__(self):
return str(self)
def process_options(options):
# tolerate sloppy auth_version
@ -186,6 +189,7 @@ _default_local_options = {
'leave_segments': False,
'changed': None,
'skip_identical': False,
'version_id': None,
'yes_all': False,
'read_acl': None,
'write_acl': None,
@ -200,6 +204,7 @@ _default_local_options = {
'meta': [],
'prefix': None,
'delimiter': None,
'versions': False,
'fail_fast': False,
'human': False,
'dir_marker': False,
@ -336,6 +341,20 @@ class SwiftPostObject(object):
self.options = options
class SwiftDeleteObject(object):
"""
Class for specifying an object delete, allowing the headers/metadata to be
specified separately for each individual object.
"""
def __init__(self, object_name, options=None):
if not (isinstance(object_name, string_types) and object_name):
raise SwiftError(
"Object names must be specified as non-empty strings"
)
self.object_name = object_name
self.options = options
class SwiftCopyObject(object):
"""
Class for specifying an object copy,
@ -489,6 +508,7 @@ class SwiftService(object):
{
'human': False,
'version_id': None,
'header': []
}
@ -871,6 +891,7 @@ class SwiftService(object):
'long': False,
'prefix': None,
'delimiter': None,
'versions': False,
'header': []
}
@ -967,13 +988,19 @@ class SwiftService(object):
@staticmethod
def _list_container_job(conn, container, options, result_queue):
marker = options.get('marker', '')
version_marker = options.get('version_marker', '')
error = None
req_headers = split_headers(options.get('header', []))
if options.get('versions', False):
query_string = 'versions=true'
else:
query_string = None
try:
while True:
_, items = conn.get_container(
container, marker=marker, prefix=options['prefix'],
delimiter=options['delimiter'], headers=req_headers
container, marker=marker, version_marker=version_marker,
prefix=options['prefix'], delimiter=options['delimiter'],
headers=req_headers, query_string=query_string
)
if not items:
@ -991,6 +1018,7 @@ class SwiftService(object):
result_queue.put(res)
marker = items[-1].get('name', items[-1].get('subdir'))
version_marker = items[-1].get('version_id', '')
except ClientException as err:
traceback, err_time = report_traceback()
logger.exception(err)
@ -1016,6 +1044,7 @@ class SwiftService(object):
'prefix': options['prefix'],
'success': False,
'marker': marker,
'version_marker': version_marker,
'error': error[0],
'traceback': error[1],
'error_timestamp': error[2]
@ -1042,6 +1071,7 @@ class SwiftService(object):
'no_download': False,
'header': [],
'skip_identical': False,
'version_id': None,
'out_directory': None,
'checksum': True,
'out_file': None,
@ -1151,6 +1181,9 @@ class SwiftService(object):
get_args = {'resp_chunk_size': DISK_BUFFER,
'headers': req_headers,
'response_dict': results_dict}
if options.get('version_id') is not None:
get_args['query_string'] = (
'version-id=%s' % options['version_id'])
if options['skip_identical']:
# Assume the file is a large object; if we're wrong, the query
# string is ignored and the If-None-Match header will trigger
@ -2337,14 +2370,28 @@ class SwiftService(object):
of objects.
:param container: The container to delete or delete from.
:param objects: The list of objects to delete.
:param objects: A list of object names (strings) or SwiftDeleteObject
instances containing an object name, and an
options dict (can be None) to override the options for
that individual delete operation::
[
'object_name',
SwiftDeleteObject('object_name',
options={...}),
...
]
The options dict is described below.
:param options: A dictionary containing options to override the global
options specified during the service object creation::
{
'yes_all': False,
'leave_segments': False,
'version_id': None,
'prefix': None,
'versions': False,
'header': [],
}
@ -2364,23 +2411,28 @@ class SwiftService(object):
if container is not None:
if objects is not None:
delete_objects = self._make_delete_objects(objects)
if options['prefix']:
objects = [obj for obj in objects
if obj.startswith(options['prefix'])]
delete_objects = [
obj for obj in delete_objects
if obj.object_name.startswith(options['prefix'])]
rq = Queue()
obj_dels = {}
bulk_page_size = self._bulk_delete_page_size(objects)
bulk_page_size = self._bulk_delete_page_size(delete_objects)
if bulk_page_size > 1:
page_at_a_time = n_at_a_time(objects, bulk_page_size)
page_at_a_time = n_at_a_time(delete_objects,
bulk_page_size)
for page_slice in page_at_a_time:
for obj_slice in n_groups(
page_slice,
self._options['object_dd_threads']):
self._bulk_delete(container, obj_slice, options,
object_names = [
obj.object_name for obj in obj_slice]
self._bulk_delete(container, object_names, options,
obj_dels)
else:
self._per_item_delete(container, objects, options,
self._per_item_delete(container, delete_objects, options,
obj_dels, rq)
# Start a thread to watch for delete results
@ -2445,6 +2497,11 @@ class SwiftService(object):
# Not many objects; may as well delete one-by-one
return 1
if any(obj.options for obj in objects
if isinstance(obj, SwiftDeleteObject)):
# we can't do per option deletes for bulk
return 1
try:
cap_result = self.capabilities()
if not cap_result['success']:
@ -2463,9 +2520,11 @@ class SwiftService(object):
return 1
def _per_item_delete(self, container, objects, options, rdict, rq):
for obj in objects:
for delete_obj in objects:
obj = delete_obj.object_name
obj_options = dict(options, **delete_obj.options or {})
obj_del = self.thread_manager.object_dd_pool.submit(
self._delete_object, container, obj, options,
self._delete_object, container, obj, obj_options,
results_queue=rq
)
obj_details = {'container': container, 'object': obj}
@ -2500,6 +2559,24 @@ class SwiftService(object):
results_queue.put(res)
return res
@staticmethod
def _make_delete_objects(objects):
delete_objects = []
for o in objects:
if isinstance(o, string_types):
obj = SwiftDeleteObject(o)
delete_objects.append(obj)
elif isinstance(o, SwiftDeleteObject):
delete_objects.append(o)
else:
raise SwiftError(
"The delete operation takes only strings or "
"SwiftDeleteObjects as input",
obj=o)
return delete_objects
def _delete_object(self, conn, container, obj, options,
results_queue=None):
_headers = {}
@ -2511,7 +2588,7 @@ class SwiftService(object):
}
try:
old_manifest = None
query_string = None
query_params = {}
if not options['leave_segments']:
try:
@ -2520,11 +2597,15 @@ class SwiftService(object):
query_string='symlink=get')
old_manifest = headers.get('x-object-manifest')
if config_true_value(headers.get('x-static-large-object')):
query_string = 'multipart-manifest=delete'
query_params['multipart-manifest'] = 'delete'
except ClientException as err:
if err.http_status != 404:
raise
if options.get('version_id') is not None:
query_params['version-id'] = options['version_id']
query_string = '&'.join('%s=%s' % (k, v) for (k, v)
in sorted(query_params.items()))
results_dict = {}
conn.delete_object(container, obj,
headers=_headers,
@ -2611,12 +2692,17 @@ class SwiftService(object):
try:
for part in self.list(container=container, options=options):
if not part["success"]:
raise part["error"]
delete_objects = []
for item in part['listing']:
delete_opts = {}
if options.get('versions', False) and 'version_id' in item:
delete_opts['version_id'] = item['version_id']
delete_obj = SwiftDeleteObject(item['name'], delete_opts)
delete_objects.append(delete_obj)
for res in self.delete(
container=container,
objects=[o['name'] for o in part['listing']],
objects=delete_objects,
options=options):
yield res
if options['prefix']:
@ -2679,7 +2765,9 @@ class SwiftService(object):
'No content received on account POST. '
'Is the bulk operations middleware enabled?')})
except Exception as e:
res.update({'success': False, 'error': e})
traceback, err_time = report_traceback()
logger.exception(e)
res.update({'success': False, 'error': e, 'traceback': traceback})
res.update({
'action': 'bulk_delete',

View File

@ -65,7 +65,8 @@ st_delete_options = '''[--all] [--leave-segments]
[--container-threads <threads>]
[--header <header:value>]
[--prefix <prefix>]
[<container> [<object>] [...]]
[--versions]
[<container> [<object>] [--version-id <version_id>] [...]]
'''
st_delete_help = '''
@ -78,6 +79,7 @@ Positional arguments:
Optional arguments:
-a, --all Delete all containers and objects.
--versions Delete all versions
--leave-segments Do not delete segments of manifest objects.
-H, --header <header:value>
Adds a custom request header to use for deleting
@ -89,6 +91,8 @@ Optional arguments:
Number of threads to use for deleting containers.
Default is 10.
--prefix <prefix> Only delete objects beginning with <prefix>.
--version-id <version-id>
Delete specific version of a versioned object.
'''.strip("\n")
@ -96,9 +100,14 @@ def st_delete(parser, args, output_manager, return_parser=False):
parser.add_argument(
'-a', '--all', action='store_true', dest='yes_all',
default=False, help='Delete all containers and objects.')
parser.add_argument('--versions', action='store_true',
help='delete all versions')
parser.add_argument(
'-p', '--prefix', dest='prefix',
help='Only delete items beginning with <prefix>.')
parser.add_argument(
'--version-id', action='store', default=None,
help='Delete a specific version of a versioned object')
parser.add_argument(
'-H', '--header', action='append', dest='header',
default=[],
@ -128,6 +137,10 @@ def st_delete(parser, args, output_manager, return_parser=False):
BASENAME, st_delete_options,
st_delete_help)
return
if options['versions'] and len(args) >= 2:
exit('--versions option not allowed for object deletes')
if options['version_id'] and len(args) < 2:
exit('--version-id option only allowed for object deletes')
if options['object_threads'] <= 0:
output_manager.error(
@ -227,6 +240,7 @@ st_download_options = '''[--all] [--marker <marker>] [--prefix <prefix>]
[--object-threads <threads>] [--ignore-checksum]
[--container-threads <threads>] [--no-download]
[--skip-identical] [--remove-prefix]
[--version-id <version_id>]
[--header <header:value>] [--no-shuffle]
[<container> [<object>] [...]]
'''
@ -271,6 +285,8 @@ Optional arguments:
Example: --header "content-type:text/plain"
--skip-identical Skip downloading files that are identical on both
sides.
--version-id <version-id>
Download specific version of a versioned object.
--ignore-checksum Turn off checksum validation for downloads.
--no-shuffle By default, when downloading a complete account or
container, download order is randomised in order to
@ -332,6 +348,9 @@ def st_download(parser, args, output_manager, return_parser=False):
'--skip-identical', action='store_true', dest='skip_identical',
default=False, help='Skip downloading files that are identical on '
'both sides.')
parser.add_argument(
'--version-id', action='store', default=None,
help='Download a specific version of a versioned object')
parser.add_argument(
'--ignore-checksum', action='store_false', dest='checksum',
default=True, help='Turn off checksum validation for downloads.')
@ -372,6 +391,8 @@ def st_download(parser, args, output_manager, return_parser=False):
output_manager.error('Usage: %s download %s\n%s', BASENAME,
st_download_options, st_download_help)
return
if options['version_id'] and len(args) < 2:
exit('--version-id option only allowed for object downloads')
if options['object_threads'] <= 0:
output_manager.error(
@ -479,7 +500,7 @@ def st_download(parser, args, output_manager, return_parser=False):
st_list_options = '''[--long] [--lh] [--totals] [--prefix <prefix>]
[--delimiter <delimiter>] [--header <header:value>]
[<container>]
[--versions] [<container>]
'''
st_list_help = '''
@ -499,6 +520,8 @@ Optional arguments:
Roll up items with the given delimiter. For containers
only. See OpenStack Swift API documentation for what
this means.
-j, --json Display listing information in json
--versions Display listing information for all versions
-H, --header <header:value>
Adds a custom request header to use for listing.
'''.strip('\n')
@ -579,6 +602,8 @@ def st_list(parser, args, output_manager, return_parser=False):
'what this means.')
parser.add_argument('-j', '--json', action='store_true',
help='print listing information in json')
parser.add_argument('--versions', action='store_true',
help='display all versions')
parser.add_argument(
'-H', '--header', action='append', dest='header',
default=[],
@ -592,6 +617,8 @@ def st_list(parser, args, output_manager, return_parser=False):
args = args[1:]
if options['delimiter'] and not args:
exit('-d option only allowed for container listings')
if options['versions'] and not args:
exit('--versions option only allowed for container listings')
human = options.pop('human')
if human:
@ -642,6 +669,7 @@ def st_list(parser, args, output_manager, return_parser=False):
st_stat_options = '''[--lh] [--header <header:value>]
[--version-id <version_id>]
[<container> [<object>]]
'''
@ -655,6 +683,8 @@ Positional arguments:
Optional arguments:
--lh Report sizes in human readable format similar to
ls -lh.
--version-id <version-id>
Report stat of specific version of a versioned object.
-H, --header <header:value>
Adds a custom request header to use for stat.
'''.strip('\n')
@ -664,6 +694,9 @@ def st_stat(parser, args, output_manager, return_parser=False):
parser.add_argument(
'--lh', dest='human', action='store_true', default=False,
help='Report sizes in human readable format similar to ls -lh.')
parser.add_argument(
'--version-id', action='store', default=None,
help='Report stat of a specific version of a versioned object')
parser.add_argument(
'-H', '--header', action='append', dest='header',
default=[],
@ -675,6 +708,8 @@ def st_stat(parser, args, output_manager, return_parser=False):
options, args = parse_args(parser, args)
args = args[1:]
if options['version_id'] and len(args) < 2:
exit('--version-id option only allowed for object stats')
with SwiftService(options=options) as swift:
try:

View File

@ -21,6 +21,7 @@ import six
import tempfile
import unittest
import time
import json
from concurrent.futures import Future
from hashlib import md5
@ -33,7 +34,7 @@ import swiftclient
import swiftclient.utils as utils
from swiftclient.client import Connection, ClientException
from swiftclient.service import (
SwiftService, SwiftError, SwiftUploadObject
SwiftService, SwiftError, SwiftUploadObject, SwiftDeleteObject
)
from test.unit import utils as test_utils
@ -315,11 +316,39 @@ class TestServiceDelete(_TestServiceBase):
mock_conn.head_object.assert_called_once_with(
'test_c', 'test_o', query_string='symlink=get', headers={})
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o', query_string=None, response_dict={},
'test_c', 'test_o', query_string='', response_dict={},
headers={}
)
self.assertEqual(expected_r, r)
@mock.patch('swiftclient.service.Connection')
def test_delete_object_version(self, mock_connection_class):
mock_conn = mock_connection_class.return_value
mock_conn.url = 'http://saio/v1/AUTH_test'
mock_conn.attempts = 0
mock_conn.head_object.return_value = {}
mock_conn.delete_object.return_value = {}
expected = {
'action': 'delete_object',
'attempts': 0,
'container': 'c',
'object': 'o',
'response_dict': {},
'success': True}
with SwiftService() as swift:
delete_results = swift.delete(
container='c', objects='o', options={
'version_id': '234567.8'})
for delete_result in delete_results:
self.assertEqual(delete_result, expected)
self.assertEqual(mock_conn.mock_calls, [
mock.call.head_object('c', 'o', headers={},
query_string='symlink=get'),
mock.call.delete_object('c', 'o', headers={},
query_string='version-id=234567.8',
response_dict={}),
])
def test_delete_object_with_headers(self):
mock_q = Queue()
mock_conn = self._get_mock_connection()
@ -338,7 +367,7 @@ class TestServiceDelete(_TestServiceBase):
'test_c', 'test_o', headers={'Skip-Middleware': 'Test'},
query_string='symlink=get')
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o', query_string=None, response_dict={},
'test_c', 'test_o', query_string='', response_dict={},
headers={'Skip-Middleware': 'Test'}
)
self.assertEqual(expected_r, r)
@ -366,7 +395,7 @@ class TestServiceDelete(_TestServiceBase):
mock_conn.head_object.assert_called_once_with(
'test_c', 'test_o', query_string='symlink=get', headers={})
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o', query_string=None, response_dict={},
'test_c', 'test_o', query_string='', response_dict={},
headers={}
)
self.assertEqual(expected_r, r)
@ -431,7 +460,7 @@ class TestServiceDelete(_TestServiceBase):
self.assertEqual(expected_r, r)
expected = [
mock.call('test_c', 'test_o', query_string=None, response_dict={},
mock.call('test_c', 'test_o', query_string='', response_dict={},
headers={}),
mock.call('manifest_c', 'test_seg_1', response_dict={}),
mock.call('manifest_c', 'test_seg_2', response_dict={})]
@ -529,6 +558,63 @@ class TestServiceDelete(_TestServiceBase):
if errors:
self.fail('_bulk_delete_page_size() failed\n' + '\n'.join(errors))
@mock.patch('swiftclient.service.Connection')
def test_bulk_delete(self, mock_connection_class):
mock_conn = mock_connection_class.return_value
mock_conn.attempts = 0
mock_conn.get_capabilities.return_value = {
'bulk_delete': {}}
stub_headers = {}
stub_resp = []
mock_conn.post_account.return_value = (
stub_headers, json.dumps(stub_resp).encode('utf8'))
obj_list = ['x%02d' % i for i in range(100)]
expected = [{
'action': u'bulk_delete',
'attempts': 0,
'container': 'c',
'objects': list(objs),
'response_dict': {},
'result': [],
'success': True,
} for objs in zip(*[iter(obj_list)] * 10)]
found_result = []
with SwiftService(options={'object_dd_threads': 10}) as swift:
delete_results = swift.delete(container='c', objects=obj_list)
for delete_result in delete_results:
found_result.append(delete_result)
self.assertEqual(sorted(found_result, key=lambda r: r['objects'][0]),
expected)
@mock.patch('swiftclient.service.Connection')
def test_bulk_delete_versions(self, mock_connection_class):
mock_conn = mock_connection_class.return_value
mock_conn.attempts = 0
mock_conn.get_capabilities.return_value = {
'bulk_delete': {}}
mock_conn.head_object.return_value = {}
stub_headers = {}
stub_resp = []
mock_conn.post_account.return_value = (
stub_headers, json.dumps(stub_resp))
obj_list = [SwiftDeleteObject('x%02d' % i, options={'version_id': i})
for i in range(100)]
expected = [{
'action': u'delete_object',
'attempts': 0,
'container': 'c',
'object': obj.object_name,
'response_dict': {},
'success': True,
} for obj in obj_list]
found_result = []
with SwiftService(options={'object_dd_threads': 10}) as swift:
delete_results = swift.delete(container='c', objects=obj_list)
for delete_result in delete_results:
found_result.append(delete_result)
self.assertEqual(sorted(found_result, key=lambda r: r['object']),
expected)
class TestSwiftError(unittest.TestCase):
@ -938,9 +1024,11 @@ class TestServiceList(_TestServiceBase):
self.assertIsNone(self._get_queue(mock_q))
self.assertEqual(mock_conn.get_container.mock_calls, [
mock.call('test_c', headers={'Skip-Middleware': 'Test'},
delimiter='', marker='', prefix=None),
delimiter='', marker='', prefix=None,
query_string=None, version_marker=''),
mock.call('test_c', headers={'Skip-Middleware': 'Test'},
delimiter='', marker='test_o', prefix=None)])
delimiter='', marker='test_o', prefix=None,
query_string=None, version_marker='')])
def test_list_container_exception(self):
mock_q = Queue()
@ -952,6 +1040,7 @@ class TestServiceList(_TestServiceBase):
'success': False,
'error': self.exc,
'marker': '',
'version_marker': '',
'error_timestamp': mock.ANY,
'traceback': mock.ANY
})
@ -961,11 +1050,61 @@ class TestServiceList(_TestServiceBase):
)
mock_conn.get_container.assert_called_once_with(
'test_c', marker='', delimiter='', prefix=None, headers={}
'test_c', marker='', delimiter='', prefix=None, headers={},
query_string=None, version_marker='',
)
self.assertEqual(expected_r, self._get_queue(mock_q))
self.assertIsNone(self._get_queue(mock_q))
@mock.patch('swiftclient.service.Connection')
def test_list_container_versions(self, mock_connection_class):
mock_conn = mock_connection_class.return_value
mock_conn.url = 'http://saio/v1/AUTH_test'
resp_headers = {}
items = [{
"bytes": 9,
"content_type": "application/octet-stream",
"hash": "e55cedc11adb39c404b7365f7d6291fa",
"is_latest": True,
"last_modified": "2019-11-08T05:00:15.115360",
"name": "test",
"version_id": "1573189215.11536"
}, {
"bytes": 8,
"content_type": "application/octet-stream",
"hash": "70c1db56f301c9e337b0099bd4174b28",
"is_latest": False,
"last_modified": "2019-11-08T05:00:14.730240",
"name": "test",
"version_id": "1573184903.06720"
}]
mock_conn.get_container.side_effect = [
(resp_headers, items),
(resp_headers, []),
]
expected = {
'action': 'list_container_part',
'container': 'c',
'listing': items,
'marker': '',
'prefix': None,
'success': True,
}
with SwiftService() as swift:
list_result_gen = swift.list(container='c', options={
'versions': True})
self.maxDiff = None
for result in list_result_gen:
self.assertEqual(result, expected)
self.assertEqual(mock_conn.get_container.mock_calls, [
mock.call('c', delimiter=None, headers={}, marker='',
prefix=None, query_string='versions=true',
version_marker=''),
mock.call('c', delimiter=None, headers={}, marker='test',
prefix=None, query_string='versions=true',
version_marker='1573184903.06720'),
])
@mock.patch('swiftclient.service.get_conn')
def test_list_queue_size(self, mock_get_conn):
mock_conn = self._get_mock_connection()
@ -1042,6 +1181,67 @@ class TestServiceList(_TestServiceBase):
self.assertEqual(observed_listing, expected_listing)
class TestServiceStat(_TestServiceBase):
maxDiff = None
@mock.patch('swiftclient.service.Connection')
def test_stat_object(self, mock_connection_class):
mock_conn = mock_connection_class.return_value
mock_conn.url = 'http://saio/v1/AUTH_test'
mock_conn.head_object.return_value = {}
expected = {
'action': 'stat_object',
'container': 'c',
'object': 'o',
'headers': {},
'items': [('Account', 'AUTH_test'),
('Container', 'c'),
('Object', 'o'),
('Content Type', None),
('Content Length', '0'),
('Last Modified', None),
('ETag', None),
('Manifest', None)],
'success': True}
with SwiftService() as swift:
stat_results = swift.stat(container='c', objects='o')
for stat_result in stat_results:
self.assertEqual(stat_result, expected)
self.assertEqual(mock_conn.head_object.mock_calls, [
mock.call('c', 'o', headers={}, query_string=None),
])
@mock.patch('swiftclient.service.Connection')
def test_stat_versioned_object(self, mock_connection_class):
mock_conn = mock_connection_class.return_value
mock_conn.url = 'http://saio/v1/AUTH_test'
mock_conn.head_object.return_value = {}
expected = {
'action': 'stat_object',
'container': 'c',
'object': 'o',
'headers': {},
'items': [('Account', 'AUTH_test'),
('Container', 'c'),
('Object', 'o'),
('Content Type', None),
('Content Length', '0'),
('Last Modified', None),
('ETag', None),
('Manifest', None)],
'success': True}
with SwiftService() as swift:
stat_results = swift.stat(container='c', objects='o', options={
'version_id': '234567.8'})
for stat_result in stat_results:
self.assertEqual(stat_result, expected)
self.assertEqual(mock_conn.head_object.mock_calls, [
mock.call('c', 'o', headers={},
query_string='version-id=234567.8'),
])
class TestService(unittest.TestCase):
def test_upload_with_bad_segment_size(self):
@ -1791,13 +1991,14 @@ class TestServiceUpload(_TestServiceBase):
mock_conn.head_object.assert_called_with('test_c', 'test_o')
expected = [
mock.call('test_c_segments', prefix='test_o/prefix',
marker='', delimiter=None, headers={}),
marker='', delimiter=None, headers={},
query_string=None, version_marker=''),
mock.call('test_c_segments', prefix='test_o/prefix',
marker="test_o/prefix/01", delimiter=None,
headers={}),
headers={}, query_string=None, version_marker=''),
mock.call('test_c_segments', prefix='test_o/prefix',
marker="test_o/prefix/02", delimiter=None,
headers={}),
headers={}, query_string=None, version_marker=''),
]
mock_conn.get_container.assert_has_calls(expected)
@ -2332,6 +2533,29 @@ class TestServiceDownload(_TestServiceBase):
self.assertEqual(resp['object'], 'test')
self.assertEqual(resp['path'], 'test')
def test_download_version_id(self):
self.opts['version_id'] = '23456.7'
with mock.patch('swiftclient.service.Connection') as mock_conn:
header = {'content-length': self.obj_len,
'etag': self.obj_etag}
mock_conn.get_object.return_value = header, self._readbody()
resp = SwiftService()._download_object_job(mock_conn,
'c',
'test',
self.opts)
self.assertIsNone(resp.get('error'))
self.assertIs(True, resp['success'])
self.assertEqual(resp['action'], 'download_object')
self.assertEqual(resp['object'], 'test')
self.assertEqual(resp['path'], 'test')
self.assertEqual(mock_conn.get_object.mock_calls, [
mock.call(
'c', 'test', headers={}, query_string='version-id=23456.7',
resp_chunk_size=65536, response_dict={}),
])
@mock.patch('swiftclient.service.interruptable_as_completed')
@mock.patch('swiftclient.service.SwiftService._download_container')
@mock.patch('swiftclient.service.SwiftService._download_object_job')
@ -2545,17 +2769,17 @@ class TestServiceDownload(_TestServiceBase):
delimiter=None,
prefix='test_o/prefix',
marker='',
headers={}),
headers={}, query_string=None, version_marker=''),
mock.call('test_c_segments',
delimiter=None,
prefix='test_o/prefix',
marker='test_o/prefix/2',
headers={}),
headers={}, query_string=None, version_marker=''),
mock.call('test_c_segments',
delimiter=None,
prefix='test_o/prefix',
marker='test_o/prefix/3',
headers={})])
headers={}, query_string=None, version_marker='')])
def test_download_object_job_skip_identical_nested_slo(self):
with tempfile.NamedTemporaryFile() as f:
@ -2682,6 +2906,7 @@ class TestServiceDownload(_TestServiceBase):
obj='test_o',
options=options)
self.maxDiff = None
self.assertEqual(r, expected_r)
self.assertEqual(mock_conn.get_container.mock_calls, [
@ -2689,17 +2914,17 @@ class TestServiceDownload(_TestServiceBase):
delimiter=None,
prefix='test_o/prefix',
marker='',
headers={}),
headers={}, query_string=None, version_marker=''),
mock.call('test_c_segments',
delimiter=None,
prefix='test_o/prefix',
marker='test_o/prefix/2',
headers={}),
headers={}, query_string=None, version_marker=''),
mock.call('test_c_segments',
delimiter=None,
prefix='test_o/prefix',
marker='test_o/prefix/3',
headers={})])
headers={}, query_string=None, version_marker='')])
self.assertEqual(mock_conn.get_object.mock_calls, [
mock.call('test_c',
'test_o',

View File

@ -241,6 +241,30 @@ class TestShell(unittest.TestCase):
self.assertEqual(connection.return_value.head_container.mock_calls, [
mock.call('container', headers={'Skip-Middleware': 'Test'})])
@mock.patch('swiftclient.service.Connection')
def test_stat_version_id(self, connection):
argv = ["", "stat", "--version-id", "1"]
with self.assertRaises(SystemExit) as caught:
swiftclient.shell.main(argv)
self.assertEqual(str(caught.exception),
"--version-id option only allowed for "
"object stats")
argv = ["", "stat", "--version-id", "1", "container"]
with self.assertRaises(SystemExit) as caught:
swiftclient.shell.main(argv)
self.assertEqual(str(caught.exception),
"--version-id option only allowed for "
"object stats")
argv = ["", "stat", "--version-id", "1", "container", "object"]
connection.return_value.head_object.return_value = {}
with CaptureOutput():
swiftclient.shell.main(argv)
self.assertEqual([mock.call('container', 'object', headers={},
query_string='version-id=1')],
connection.return_value.head_object.mock_calls)
@mock.patch('swiftclient.service.Connection')
def test_stat_object(self, connection):
return_headers = {
@ -295,7 +319,45 @@ class TestShell(unittest.TestCase):
' Manifest: manifest\n')
self.assertEqual(connection.return_value.head_object.mock_calls, [
mock.call('container', 'object',
headers={'Skip-Middleware': 'Test'})])
headers={'Skip-Middleware': 'Test'},
query_string=None)])
def test_list_account_with_delimiter(self):
argv = ["", "list", "--delimiter", "foo"]
with self.assertRaises(SystemExit) as caught:
swiftclient.shell.main(argv)
self.assertEqual(str(caught.exception),
"-d option only allowed for "
"container listings")
@mock.patch('swiftclient.service.Connection')
def test_list_container_with_versions(self, connection):
connection.return_value.get_container.side_effect = [
[None, [
{'name': 'foo', 'version_id': '2'},
{'name': 'foo', 'version_id': '1'},
]],
[None, []],
]
argv = ["", "list", "container", "--versions"]
with CaptureOutput(suppress_systemexit=True) as output:
swiftclient.shell.main(argv)
calls = [mock.call('container', delimiter=None, headers={}, marker='',
prefix=None, query_string='versions=true',
version_marker=''),
mock.call('container', delimiter=None, headers={},
marker='foo', prefix=None,
query_string='versions=true', version_marker='1')]
connection.return_value.get_container.assert_has_calls(calls)
self.assertEqual(output.out, 'foo\nfoo\n')
def test_list_account_with_versions(self):
argv = ["", "list", "--versions"]
with self.assertRaises(SystemExit) as caught:
swiftclient.shell.main(argv)
self.assertEqual(str(caught.exception),
"--versions option only allowed for "
"container listings")
@mock.patch('swiftclient.service.Connection')
def test_list_json(self, connection):
@ -431,9 +493,11 @@ class TestShell(unittest.TestCase):
swiftclient.shell.main(argv)
calls = [
mock.call('container', marker='',
delimiter=None, prefix=None, headers={}),
delimiter=None, prefix=None, headers={},
query_string=None, version_marker=''),
mock.call('container', marker='object_a',
delimiter=None, prefix=None, headers={})]
delimiter=None, prefix=None, headers={},
query_string=None, version_marker='')]
connection.return_value.get_container.assert_has_calls(calls)
self.assertEqual(output.out, 'object_a\n')
@ -450,9 +514,11 @@ class TestShell(unittest.TestCase):
swiftclient.shell.main(argv)
calls = [
mock.call('container', marker='',
delimiter=None, prefix=None, headers={}),
delimiter=None, prefix=None, headers={},
query_string=None, version_marker=''),
mock.call('container', marker='object_a',
delimiter=None, prefix=None, headers={})]
delimiter=None, prefix=None, headers={},
query_string=None, version_marker='')]
connection.return_value.get_container.assert_has_calls(calls)
self.assertEqual(output.out,
@ -472,14 +538,44 @@ class TestShell(unittest.TestCase):
calls = [
mock.call('container', marker='',
delimiter=None, prefix=None,
headers={'Skip-Middleware': 'Test'}),
headers={'Skip-Middleware': 'Test'},
query_string=None, version_marker=''),
mock.call('container', marker='object_a',
delimiter=None, prefix=None,
headers={'Skip-Middleware': 'Test'})]
headers={'Skip-Middleware': 'Test'},
query_string=None, version_marker='')]
connection.return_value.get_container.assert_has_calls(calls)
self.assertEqual(output.out, 'object_a\n')
@mock.patch('swiftclient.service.Connection')
def test_download_version_id(self, connection):
argv = ["", "download", "--yes-all", "--version-id", "5"]
with self.assertRaises(SystemExit) as caught:
swiftclient.shell.main(argv)
self.assertEqual(str(caught.exception),
"--version-id option only allowed for "
"object downloads")
argv = ["", "download", "--version-id", "2", "container"]
with self.assertRaises(SystemExit) as caught:
swiftclient.shell.main(argv)
self.assertEqual(str(caught.exception),
"--version-id option only allowed for "
"object downloads")
argv = ["", "download", "--version-id", "1", "container", "object"]
connection.return_value.head_object.return_value = {}
connection.return_value.get_object.return_value = {}, ''
connection.return_value.attempts = 0
with CaptureOutput():
swiftclient.shell.main(argv)
self.assertEqual([mock.call('container', 'object', headers={},
query_string='version-id=1',
resp_chunk_size=65536,
response_dict={})],
connection.return_value.get_object.mock_calls)
@mock.patch('swiftclient.service.makedirs')
@mock.patch('swiftclient.service.Connection')
def test_download(self, connection, makedirs):
@ -1085,6 +1181,33 @@ class TestShell(unittest.TestCase):
check_good(["--object-threads", "1"])
check_good(["--container-threads", "1"])
@mock.patch('swiftclient.service.Connection')
def test_delete_version_id(self, connection):
argv = ["", "delete", "--yes-all", "--version-id", "3"]
with self.assertRaises(SystemExit) as caught:
swiftclient.shell.main(argv)
self.assertEqual(str(caught.exception),
"--version-id option only allowed for "
"object deletes")
argv = ["", "delete", "--version-id", "1", "container"]
with self.assertRaises(SystemExit) as caught:
swiftclient.shell.main(argv)
self.assertEqual(str(caught.exception),
"--version-id option only allowed for "
"object deletes")
argv = ["", "delete", "--version-id", "1", "container", "object"]
connection.return_value.head_object.return_value = {}
connection.return_value.delete_object.return_value = None
connection.return_value.attempts = 0
with CaptureOutput():
swiftclient.shell.main(argv)
self.assertEqual([mock.call('container', 'object', headers={},
query_string='version-id=1',
response_dict={})],
connection.return_value.delete_object.mock_calls)
@mock.patch.object(swiftclient.service.SwiftService,
'_bulk_delete_page_size', lambda *a: 1)
@mock.patch('swiftclient.service.Connection')
@ -1094,10 +1217,12 @@ class TestShell(unittest.TestCase):
[None, [{'name': 'empty_container'}]],
[None, []],
]
# N.B: missing --versions flag, version-id gets ignored
# only latest object is deleted
connection.return_value.get_container.side_effect = [
[None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}]],
[None, []],
[None, [{'name': 'object'}]],
[None, [{'name': 'object', 'version_id': 1}]],
[None, []],
[None, []],
]
@ -1107,11 +1232,48 @@ class TestShell(unittest.TestCase):
connection.return_value.delete_object.return_value = None
swiftclient.shell.main(argv)
connection.return_value.delete_object.assert_has_calls([
mock.call('container', 'object', query_string=None,
mock.call('container', 'object', query_string='',
response_dict={}, headers={}),
mock.call('container', 'obj\xe9ct2', query_string=None,
mock.call('container', 'obj\xe9ct2', query_string='',
response_dict={}, headers={}),
mock.call('container2', 'object', query_string=None,
mock.call('container2', 'object', query_string='',
response_dict={}, headers={})], any_order=True)
self.assertEqual(3, connection.return_value.delete_object.call_count,
'Expected 3 calls but found\n%r'
% connection.return_value.delete_object.mock_calls)
self.assertEqual(
connection.return_value.delete_container.mock_calls, [
mock.call('container', response_dict={}, headers={}),
mock.call('container2', response_dict={}, headers={}),
mock.call('empty_container', response_dict={}, headers={})])
@mock.patch.object(swiftclient.service.SwiftService,
'_bulk_delete_page_size', lambda *a: 1)
@mock.patch('swiftclient.service.Connection')
def test_delete_account_versions(self, connection):
connection.return_value.get_account.side_effect = [
[None, [{'name': 'container'}, {'name': 'container2'}]],
[None, [{'name': 'empty_container'}]],
[None, []],
]
connection.return_value.get_container.side_effect = [
[None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}]],
[None, []],
[None, [{'name': 'obj', 'version_id': 1}]],
[None, []],
[None, []],
]
connection.return_value.attempts = 0
argv = ["", "delete", "--all", "--versions"]
connection.return_value.head_object.return_value = {}
connection.return_value.delete_object.return_value = None
swiftclient.shell.main(argv)
connection.return_value.delete_object.assert_has_calls([
mock.call('container', 'object', query_string='',
response_dict={}, headers={}),
mock.call('container', 'obj\xe9ct2', query_string='',
response_dict={}, headers={}),
mock.call(