object versioning features

* add --versions to list
 * add --versions to delete
 * add --version-id to stat
 * add --version-id to delete
 * add --version-id to download

Change-Id: I89802064921778fee7efe57c7d60c976cdde3a27

swiftclient/client.py

@@ -921,7 +921,7 @@ def post_account(url, token, headers, http_conn=None, response_dict=None,
 
 def get_container(url, token, container, marker=None, limit=None,
                   prefix=None, delimiter=None, end_marker=None,
-                  path=None, http_conn=None,
+                  version_marker=None, path=None, http_conn=None,
                   full_listing=False, service_token=None, headers=None,
                   query_string=None):
     """
@@ -935,6 +935,7 @@ def get_container(url, token, container, marker=None, limit=None,
     :param prefix: prefix query
     :param delimiter: string to delimit the queries on
     :param end_marker: marker query
+    :param version_marker: version marker query
     :param path: path query (equivalent: "delimiter=/" and "prefix=path/")
     :param http_conn: a tuple of (parsed url, HTTPConnection object),
                       (If None, it will create the conn object)
@@ -951,17 +952,20 @@ def get_container(url, token, container, marker=None, limit=None,
         http_conn = http_connection(url)
     if full_listing:
         rv = get_container(url, token, container, marker, limit, prefix,
-                           delimiter, end_marker, path, http_conn,
-                           service_token=service_token, headers=headers)
+                           delimiter, end_marker, version_marker, path=path,
+                           http_conn=http_conn, service_token=service_token,
+                           headers=headers)
         listing = rv[1]
         while listing:
             if not delimiter:
                 marker = listing[-1]['name']
             else:
                 marker = listing[-1].get('name', listing[-1].get('subdir'))
+            version_marker = listing[-1].get('version_id')
             listing = get_container(url, token, container, marker, limit,
-                                    prefix, delimiter, end_marker, path,
-                                    http_conn, service_token=service_token,
+                                    prefix, delimiter, end_marker,
+                                    version_marker, path, http_conn,
+                                    service_token=service_token,
                                     headers=headers)[1]
             if listing:
                 rv[1].extend(listing)
@@ -979,6 +983,8 @@ def get_container(url, token, container, marker=None, limit=None,
         qs += '&delimiter=%s' % quote(delimiter)
     if end_marker:
         qs += '&end_marker=%s' % quote(end_marker)
+    if version_marker:
+        qs += '&version_marker=%s' % quote(version_marker)
     if path:
         qs += '&path=%s' % quote(path)
     if query_string:
@@ -1816,15 +1822,17 @@ class Connection(object):
         return self._retry(None, head_container, container, headers=headers)
 
     def get_container(self, container, marker=None, limit=None, prefix=None,
-                      delimiter=None, end_marker=None, path=None,
-                      full_listing=False, headers=None, query_string=None):
+                      delimiter=None, end_marker=None, version_marker=None,
+                      path=None, full_listing=False, headers=None,
+                      query_string=None):
         """Wrapper for :func:`get_container`"""
         # TODO(unknown): With full_listing=True this will restart the entire
         # listing with each retry. Need to make a better version that just
         # retries where it left off.
         return self._retry(None, get_container, container, marker=marker,
                            limit=limit, prefix=prefix, delimiter=delimiter,
-                           end_marker=end_marker, path=path,
+                           end_marker=end_marker,
+                           version_marker=version_marker, path=path,
                            full_listing=full_listing, headers=headers,
                            query_string=query_string)
 

swiftclient/command_helpers.py

@@ -143,7 +143,11 @@ def print_container_stats(items, headers, output_manager):
 def stat_object(conn, options, container, obj):
     req_headers = split_request_headers(options.get('header', []))
 
-    headers = conn.head_object(container, obj, headers=req_headers)
+    query_string = None
+    if options.get('version_id') is not None:
+        query_string = 'version-id=%s' % options['version_id']
+    headers = conn.head_object(container, obj, headers=req_headers,
+                               query_string=query_string)
     items = []
     if options['verbose'] > 1:
         path = '%s/%s/%s' % (conn.url, container, obj)

swiftclient/service.py

@@ -86,6 +86,9 @@ class SwiftError(Exception):
             value += " segment:%s" % self.segment
         return value
 
+    def __repr__(self):
+        return str(self)
+
 
 def process_options(options):
     # tolerate sloppy auth_version
@@ -186,6 +189,7 @@ _default_local_options = {
     'leave_segments': False,
     'changed': None,
     'skip_identical': False,
+    'version_id': None,
     'yes_all': False,
     'read_acl': None,
     'write_acl': None,
@@ -200,6 +204,7 @@ _default_local_options = {
     'meta': [],
     'prefix': None,
     'delimiter': None,
+    'versions': False,
     'fail_fast': False,
     'human': False,
     'dir_marker': False,
@@ -336,6 +341,20 @@ class SwiftPostObject(object):
         self.options = options
 
 
+class SwiftDeleteObject(object):
+    """
+    Class for specifying an object delete, allowing the headers/metadata to be
+    specified separately for each individual object.
+    """
+    def __init__(self, object_name, options=None):
+        if not (isinstance(object_name, string_types) and object_name):
+            raise SwiftError(
+                "Object names must be specified as non-empty strings"
+            )
+        self.object_name = object_name
+        self.options = options
+
+
 class SwiftCopyObject(object):
     """
     Class for specifying an object copy,
@@ -489,6 +508,7 @@ class SwiftService(object):
 
                             {
                                 'human': False,
+                                'version_id': None,
                                 'header': []
                             }
 
@@ -871,6 +891,7 @@ class SwiftService(object):
                                 'long': False,
                                 'prefix': None,
                                 'delimiter': None,
+                                'versions': False,
                                 'header': []
                             }
 
@@ -967,13 +988,19 @@ class SwiftService(object):
     @staticmethod
     def _list_container_job(conn, container, options, result_queue):
         marker = options.get('marker', '')
+        version_marker = options.get('version_marker', '')
         error = None
         req_headers = split_headers(options.get('header', []))
+        if options.get('versions', False):
+            query_string = 'versions=true'
+        else:
+            query_string = None
         try:
             while True:
                 _, items = conn.get_container(
-                    container, marker=marker, prefix=options['prefix'],
-                    delimiter=options['delimiter'], headers=req_headers
+                    container, marker=marker, version_marker=version_marker,
+                    prefix=options['prefix'], delimiter=options['delimiter'],
+                    headers=req_headers, query_string=query_string
                 )
 
                 if not items:
@@ -991,6 +1018,7 @@ class SwiftService(object):
                 result_queue.put(res)
 
                 marker = items[-1].get('name', items[-1].get('subdir'))
+                version_marker = items[-1].get('version_id', '')
         except ClientException as err:
             traceback, err_time = report_traceback()
             logger.exception(err)
@@ -1016,6 +1044,7 @@ class SwiftService(object):
             'prefix': options['prefix'],
             'success': False,
             'marker': marker,
+            'version_marker': version_marker,
             'error': error[0],
             'traceback': error[1],
             'error_timestamp': error[2]
@@ -1042,6 +1071,7 @@ class SwiftService(object):
                                 'no_download': False,
                                 'header': [],
                                 'skip_identical': False,
+                                'version_id': None,
                                 'out_directory': None,
                                 'checksum': True,
                                 'out_file': None,
@@ -1151,6 +1181,9 @@ class SwiftService(object):
             get_args = {'resp_chunk_size': DISK_BUFFER,
                         'headers': req_headers,
                         'response_dict': results_dict}
+            if options.get('version_id') is not None:
+                get_args['query_string'] = (
+                    'version-id=%s' % options['version_id'])
             if options['skip_identical']:
                 # Assume the file is a large object; if we're wrong, the query
                 # string is ignored and the If-None-Match header will trigger
@@ -2337,14 +2370,28 @@ class SwiftService(object):
         of objects.
 
         :param container: The container to delete or delete from.
-        :param objects: The list of objects to delete.
+        :param objects: A list of object names (strings) or SwiftDeleteObject
+                        instances containing an object name, and an
+                        options dict (can be None) to override the options for
+                        that individual delete operation::
+
+                            [
+                                'object_name',
+                                SwiftDeleteObject('object_name',
+                                                  options={...}),
+                                ...
+                            ]
+
+                        The options dict is described below.
         :param options: A dictionary containing options to override the global
                         options specified during the service object creation::
 
                             {
                                 'yes_all': False,
                                 'leave_segments': False,
+                                'version_id': None,
                                 'prefix': None,
+                                'versions': False,
                                 'header': [],
                             }
 
@@ -2364,23 +2411,28 @@ class SwiftService(object):
 
         if container is not None:
             if objects is not None:
+                delete_objects = self._make_delete_objects(objects)
                 if options['prefix']:
-                    objects = [obj for obj in objects
-                               if obj.startswith(options['prefix'])]
+                    delete_objects = [
+                        obj for obj in delete_objects
+                        if obj.object_name.startswith(options['prefix'])]
                 rq = Queue()
                 obj_dels = {}
 
-                bulk_page_size = self._bulk_delete_page_size(objects)
+                bulk_page_size = self._bulk_delete_page_size(delete_objects)
                 if bulk_page_size > 1:
-                    page_at_a_time = n_at_a_time(objects, bulk_page_size)
+                    page_at_a_time = n_at_a_time(delete_objects,
+                                                 bulk_page_size)
                     for page_slice in page_at_a_time:
                         for obj_slice in n_groups(
                                 page_slice,
                                 self._options['object_dd_threads']):
-                            self._bulk_delete(container, obj_slice, options,
+                            object_names = [
+                                obj.object_name for obj in obj_slice]
+                            self._bulk_delete(container, object_names, options,
                                               obj_dels)
                 else:
-                    self._per_item_delete(container, objects, options,
+                    self._per_item_delete(container, delete_objects, options,
                                           obj_dels, rq)
 
                 # Start a thread to watch for delete results
@@ -2445,6 +2497,11 @@ class SwiftService(object):
             # Not many objects; may as well delete one-by-one
             return 1
 
+        if any(obj.options for obj in objects
+               if isinstance(obj, SwiftDeleteObject)):
+            # we can't do per option deletes for bulk
+            return 1
+
         try:
             cap_result = self.capabilities()
             if not cap_result['success']:
@@ -2463,9 +2520,11 @@ class SwiftService(object):
             return 1
 
     def _per_item_delete(self, container, objects, options, rdict, rq):
-        for obj in objects:
+        for delete_obj in objects:
+            obj = delete_obj.object_name
+            obj_options = dict(options, **delete_obj.options or {})
             obj_del = self.thread_manager.object_dd_pool.submit(
-                self._delete_object, container, obj, options,
+                self._delete_object, container, obj, obj_options,
                 results_queue=rq
             )
             obj_details = {'container': container, 'object': obj}
@@ -2500,6 +2559,24 @@ class SwiftService(object):
             results_queue.put(res)
         return res
 
+    @staticmethod
+    def _make_delete_objects(objects):
+        delete_objects = []
+
+        for o in objects:
+            if isinstance(o, string_types):
+                obj = SwiftDeleteObject(o)
+                delete_objects.append(obj)
+            elif isinstance(o, SwiftDeleteObject):
+                delete_objects.append(o)
+            else:
+                raise SwiftError(
+                    "The delete operation takes only strings or "
+                    "SwiftDeleteObjects as input",
+                    obj=o)
+
+        return delete_objects
+
     def _delete_object(self, conn, container, obj, options,
                        results_queue=None):
         _headers = {}
@@ -2511,7 +2588,7 @@ class SwiftService(object):
         }
         try:
             old_manifest = None
-            query_string = None
+            query_params = {}
 
             if not options['leave_segments']:
                 try:
@@ -2520,11 +2597,15 @@ class SwiftService(object):
                                                query_string='symlink=get')
                     old_manifest = headers.get('x-object-manifest')
                     if config_true_value(headers.get('x-static-large-object')):
-                        query_string = 'multipart-manifest=delete'
+                        query_params['multipart-manifest'] = 'delete'
                 except ClientException as err:
                     if err.http_status != 404:
                         raise
 
+            if options.get('version_id') is not None:
+                query_params['version-id'] = options['version_id']
+            query_string = '&'.join('%s=%s' % (k, v) for (k, v)
+                                    in sorted(query_params.items()))
             results_dict = {}
             conn.delete_object(container, obj,
                                headers=_headers,
@@ -2611,12 +2692,17 @@ class SwiftService(object):
         try:
             for part in self.list(container=container, options=options):
                 if not part["success"]:
-
                     raise part["error"]
-
+                delete_objects = []
+                for item in part['listing']:
+                    delete_opts = {}
+                    if options.get('versions', False) and 'version_id' in item:
+                        delete_opts['version_id'] = item['version_id']
+                    delete_obj = SwiftDeleteObject(item['name'], delete_opts)
+                    delete_objects.append(delete_obj)
                 for res in self.delete(
                         container=container,
-                        objects=[o['name'] for o in part['listing']],
+                        objects=delete_objects,
                         options=options):
                     yield res
             if options['prefix']:
@@ -2679,7 +2765,9 @@ class SwiftService(object):
                         'No content received on account POST. '
                         'Is the bulk operations middleware enabled?')})
         except Exception as e:
-            res.update({'success': False, 'error': e})
+            traceback, err_time = report_traceback()
+            logger.exception(e)
+            res.update({'success': False, 'error': e, 'traceback': traceback})
 
         res.update({
             'action': 'bulk_delete',

swiftclient/shell.py

@@ -65,7 +65,8 @@ st_delete_options = '''[--all] [--leave-segments]
                     [--container-threads <threads>]
                     [--header <header:value>]
                     [--prefix <prefix>]
-                    [<container> [<object>] [...]]
+                    [--versions]
+                    [<container> [<object>] [--version-id <version_id>] [...]]
 '''
 
 st_delete_help = '''
@@ -78,6 +79,7 @@ Positional arguments:
 
 Optional arguments:
   -a, --all             Delete all containers and objects.
+  --versions            Delete all versions
   --leave-segments      Do not delete segments of manifest objects.
   -H, --header <header:value>
                         Adds a custom request header to use for deleting
@@ -89,6 +91,8 @@ Optional arguments:
                         Number of threads to use for deleting containers.
                         Default is 10.
   --prefix <prefix>     Only delete objects beginning with <prefix>.
+  --version-id <version-id>
+                        Delete specific version of a versioned object.
 '''.strip("\n")
 
 
@@ -96,9 +100,14 @@ def st_delete(parser, args, output_manager, return_parser=False):
     parser.add_argument(
         '-a', '--all', action='store_true', dest='yes_all',
         default=False, help='Delete all containers and objects.')
+    parser.add_argument('--versions', action='store_true',
+                        help='delete all versions')
     parser.add_argument(
         '-p', '--prefix', dest='prefix',
         help='Only delete items beginning with <prefix>.')
+    parser.add_argument(
+        '--version-id', action='store', default=None,
+        help='Delete a specific version of a versioned object')
     parser.add_argument(
         '-H', '--header', action='append', dest='header',
         default=[],
@@ -128,6 +137,10 @@ def st_delete(parser, args, output_manager, return_parser=False):
                              BASENAME, st_delete_options,
                              st_delete_help)
         return
+    if options['versions'] and len(args) >= 2:
+        exit('--versions option not allowed for object deletes')
+    if options['version_id'] and len(args) < 2:
+        exit('--version-id option only allowed for object deletes')
 
     if options['object_threads'] <= 0:
         output_manager.error(
@@ -227,6 +240,7 @@ st_download_options = '''[--all] [--marker <marker>] [--prefix <prefix>]
                       [--object-threads <threads>] [--ignore-checksum]
                       [--container-threads <threads>] [--no-download]
                       [--skip-identical] [--remove-prefix]
+                      [--version-id <version_id>]
                       [--header <header:value>] [--no-shuffle]
                       [<container> [<object>] [...]]
 '''
@@ -271,6 +285,8 @@ Optional arguments:
                         Example: --header "content-type:text/plain"
   --skip-identical      Skip downloading files that are identical on both
                         sides.
+  --version-id <version-id>
+                        Download specific version of a versioned object.
   --ignore-checksum     Turn off checksum validation for downloads.
   --no-shuffle          By default, when downloading a complete account or
                         container, download order is randomised in order to
@@ -332,6 +348,9 @@ def st_download(parser, args, output_manager, return_parser=False):
         '--skip-identical', action='store_true', dest='skip_identical',
         default=False, help='Skip downloading files that are identical on '
         'both sides.')
+    parser.add_argument(
+        '--version-id', action='store', default=None,
+        help='Download a specific version of a versioned object')
     parser.add_argument(
         '--ignore-checksum', action='store_false', dest='checksum',
         default=True, help='Turn off checksum validation for downloads.')
@@ -372,6 +391,8 @@ def st_download(parser, args, output_manager, return_parser=False):
         output_manager.error('Usage: %s download %s\n%s', BASENAME,
                              st_download_options, st_download_help)
         return
+    if options['version_id'] and len(args) < 2:
+        exit('--version-id option only allowed for object downloads')
 
     if options['object_threads'] <= 0:
         output_manager.error(
@@ -479,7 +500,7 @@ def st_download(parser, args, output_manager, return_parser=False):
 
 st_list_options = '''[--long] [--lh] [--totals] [--prefix <prefix>]
                   [--delimiter <delimiter>] [--header <header:value>]
-                  [<container>]
+                  [--versions] [<container>]
 '''
 
 st_list_help = '''
@@ -499,6 +520,8 @@ Optional arguments:
                         Roll up items with the given delimiter. For containers
                         only. See OpenStack Swift API documentation for what
                         this means.
+  -j, --json            Display listing information in json
+  --versions            Display listing information for all versions
   -H, --header <header:value>
                         Adds a custom request header to use for listing.
 '''.strip('\n')
@@ -579,6 +602,8 @@ def st_list(parser, args, output_manager, return_parser=False):
              'what this means.')
     parser.add_argument('-j', '--json', action='store_true',
                         help='print listing information in json')
+    parser.add_argument('--versions', action='store_true',
+                        help='display all versions')
     parser.add_argument(
         '-H', '--header', action='append', dest='header',
         default=[],
@@ -592,6 +617,8 @@ def st_list(parser, args, output_manager, return_parser=False):
     args = args[1:]
     if options['delimiter'] and not args:
         exit('-d option only allowed for container listings')
+    if options['versions'] and not args:
+        exit('--versions option only allowed for container listings')
 
     human = options.pop('human')
     if human:
@@ -642,6 +669,7 @@ def st_list(parser, args, output_manager, return_parser=False):
 
 
 st_stat_options = '''[--lh] [--header <header:value>]
+                  [--version-id <version_id>]
                   [<container> [<object>]]
 '''
 
@@ -655,6 +683,8 @@ Positional arguments:
 Optional arguments:
   --lh                  Report sizes in human readable format similar to
                         ls -lh.
+  --version-id <version-id>
+                        Report stat of specific version of a versioned object.
   -H, --header <header:value>
                         Adds a custom request header to use for stat.
 '''.strip('\n')
@@ -664,6 +694,9 @@ def st_stat(parser, args, output_manager, return_parser=False):
     parser.add_argument(
         '--lh', dest='human', action='store_true', default=False,
         help='Report sizes in human readable format similar to ls -lh.')
+    parser.add_argument(
+        '--version-id', action='store', default=None,
+        help='Report stat of a specific version of a versioned object')
     parser.add_argument(
         '-H', '--header', action='append', dest='header',
         default=[],
@@ -675,6 +708,8 @@ def st_stat(parser, args, output_manager, return_parser=False):
 
     options, args = parse_args(parser, args)
     args = args[1:]
+    if options['version_id'] and len(args) < 2:
+        exit('--version-id option only allowed for object stats')
 
     with SwiftService(options=options) as swift:
         try:

test/unit/test_service.py

@@ -21,6 +21,7 @@ import six
 import tempfile
 import unittest
 import time
+import json
 
 from concurrent.futures import Future
 from hashlib import md5
@@ -33,7 +34,7 @@ import swiftclient
 import swiftclient.utils as utils
 from swiftclient.client import Connection, ClientException
 from swiftclient.service import (
-    SwiftService, SwiftError, SwiftUploadObject
+    SwiftService, SwiftError, SwiftUploadObject, SwiftDeleteObject
 )
 
 from test.unit import utils as test_utils
@@ -315,11 +316,39 @@ class TestServiceDelete(_TestServiceBase):
         mock_conn.head_object.assert_called_once_with(
             'test_c', 'test_o', query_string='symlink=get', headers={})
         mock_conn.delete_object.assert_called_once_with(
-            'test_c', 'test_o', query_string=None, response_dict={},
+            'test_c', 'test_o', query_string='', response_dict={},
             headers={}
         )
         self.assertEqual(expected_r, r)
 
+    @mock.patch('swiftclient.service.Connection')
+    def test_delete_object_version(self, mock_connection_class):
+        mock_conn = mock_connection_class.return_value
+        mock_conn.url = 'http://saio/v1/AUTH_test'
+        mock_conn.attempts = 0
+        mock_conn.head_object.return_value = {}
+        mock_conn.delete_object.return_value = {}
+        expected = {
+            'action': 'delete_object',
+            'attempts': 0,
+            'container': 'c',
+            'object': 'o',
+            'response_dict': {},
+            'success': True}
+        with SwiftService() as swift:
+            delete_results = swift.delete(
+                container='c', objects='o', options={
+                    'version_id': '234567.8'})
+            for delete_result in delete_results:
+                self.assertEqual(delete_result, expected)
+        self.assertEqual(mock_conn.mock_calls, [
+            mock.call.head_object('c', 'o', headers={},
+                                  query_string='symlink=get'),
+            mock.call.delete_object('c', 'o', headers={},
+                                    query_string='version-id=234567.8',
+                                    response_dict={}),
+        ])
+
     def test_delete_object_with_headers(self):
         mock_q = Queue()
         mock_conn = self._get_mock_connection()
@@ -338,7 +367,7 @@ class TestServiceDelete(_TestServiceBase):
             'test_c', 'test_o', headers={'Skip-Middleware': 'Test'},
             query_string='symlink=get')
         mock_conn.delete_object.assert_called_once_with(
-            'test_c', 'test_o', query_string=None, response_dict={},
+            'test_c', 'test_o', query_string='', response_dict={},
             headers={'Skip-Middleware': 'Test'}
         )
         self.assertEqual(expected_r, r)
@@ -366,7 +395,7 @@ class TestServiceDelete(_TestServiceBase):
         mock_conn.head_object.assert_called_once_with(
             'test_c', 'test_o', query_string='symlink=get', headers={})
         mock_conn.delete_object.assert_called_once_with(
-            'test_c', 'test_o', query_string=None, response_dict={},
+            'test_c', 'test_o', query_string='', response_dict={},
             headers={}
         )
         self.assertEqual(expected_r, r)
@@ -431,7 +460,7 @@ class TestServiceDelete(_TestServiceBase):
 
         self.assertEqual(expected_r, r)
         expected = [
-            mock.call('test_c', 'test_o', query_string=None, response_dict={},
+            mock.call('test_c', 'test_o', query_string='', response_dict={},
                       headers={}),
             mock.call('manifest_c', 'test_seg_1', response_dict={}),
             mock.call('manifest_c', 'test_seg_2', response_dict={})]
@@ -529,6 +558,63 @@ class TestServiceDelete(_TestServiceBase):
         if errors:
             self.fail('_bulk_delete_page_size() failed\n' + '\n'.join(errors))
 
+    @mock.patch('swiftclient.service.Connection')
+    def test_bulk_delete(self, mock_connection_class):
+        mock_conn = mock_connection_class.return_value
+        mock_conn.attempts = 0
+        mock_conn.get_capabilities.return_value = {
+            'bulk_delete': {}}
+        stub_headers = {}
+        stub_resp = []
+        mock_conn.post_account.return_value = (
+            stub_headers, json.dumps(stub_resp).encode('utf8'))
+        obj_list = ['x%02d' % i for i in range(100)]
+        expected = [{
+            'action': u'bulk_delete',
+            'attempts': 0,
+            'container': 'c',
+            'objects': list(objs),
+            'response_dict': {},
+            'result': [],
+            'success': True,
+        } for objs in zip(*[iter(obj_list)] * 10)]
+        found_result = []
+        with SwiftService(options={'object_dd_threads': 10}) as swift:
+            delete_results = swift.delete(container='c', objects=obj_list)
+            for delete_result in delete_results:
+                found_result.append(delete_result)
+        self.assertEqual(sorted(found_result, key=lambda r: r['objects'][0]),
+                         expected)
+
+    @mock.patch('swiftclient.service.Connection')
+    def test_bulk_delete_versions(self, mock_connection_class):
+        mock_conn = mock_connection_class.return_value
+        mock_conn.attempts = 0
+        mock_conn.get_capabilities.return_value = {
+            'bulk_delete': {}}
+        mock_conn.head_object.return_value = {}
+        stub_headers = {}
+        stub_resp = []
+        mock_conn.post_account.return_value = (
+            stub_headers, json.dumps(stub_resp))
+        obj_list = [SwiftDeleteObject('x%02d' % i, options={'version_id': i})
+                    for i in range(100)]
+        expected = [{
+            'action': u'delete_object',
+            'attempts': 0,
+            'container': 'c',
+            'object': obj.object_name,
+            'response_dict': {},
+            'success': True,
+        } for obj in obj_list]
+        found_result = []
+        with SwiftService(options={'object_dd_threads': 10}) as swift:
+            delete_results = swift.delete(container='c', objects=obj_list)
+            for delete_result in delete_results:
+                found_result.append(delete_result)
+        self.assertEqual(sorted(found_result, key=lambda r: r['object']),
+                         expected)
+
 
 class TestSwiftError(unittest.TestCase):
 
@@ -938,9 +1024,11 @@ class TestServiceList(_TestServiceBase):
         self.assertIsNone(self._get_queue(mock_q))
         self.assertEqual(mock_conn.get_container.mock_calls, [
             mock.call('test_c', headers={'Skip-Middleware': 'Test'},
-                      delimiter='', marker='', prefix=None),
+                      delimiter='', marker='', prefix=None,
+                      query_string=None, version_marker=''),
             mock.call('test_c', headers={'Skip-Middleware': 'Test'},
-                      delimiter='', marker='test_o', prefix=None)])
+                      delimiter='', marker='test_o', prefix=None,
+                      query_string=None, version_marker='')])
 
     def test_list_container_exception(self):
         mock_q = Queue()
@@ -952,6 +1040,7 @@ class TestServiceList(_TestServiceBase):
             'success': False,
             'error': self.exc,
             'marker': '',
+            'version_marker': '',
             'error_timestamp': mock.ANY,
             'traceback': mock.ANY
         })
@@ -961,11 +1050,61 @@ class TestServiceList(_TestServiceBase):
         )
 
         mock_conn.get_container.assert_called_once_with(
-            'test_c', marker='', delimiter='', prefix=None, headers={}
+            'test_c', marker='', delimiter='', prefix=None, headers={},
+            query_string=None, version_marker='',
         )
         self.assertEqual(expected_r, self._get_queue(mock_q))
         self.assertIsNone(self._get_queue(mock_q))
 
+    @mock.patch('swiftclient.service.Connection')
+    def test_list_container_versions(self, mock_connection_class):
+        mock_conn = mock_connection_class.return_value
+        mock_conn.url = 'http://saio/v1/AUTH_test'
+        resp_headers = {}
+        items = [{
+            "bytes": 9,
+            "content_type": "application/octet-stream",
+            "hash": "e55cedc11adb39c404b7365f7d6291fa",
+            "is_latest": True,
+            "last_modified": "2019-11-08T05:00:15.115360",
+            "name": "test",
+            "version_id": "1573189215.11536"
+        }, {
+            "bytes": 8,
+            "content_type": "application/octet-stream",
+            "hash": "70c1db56f301c9e337b0099bd4174b28",
+            "is_latest": False,
+            "last_modified": "2019-11-08T05:00:14.730240",
+            "name": "test",
+            "version_id": "1573184903.06720"
+        }]
+        mock_conn.get_container.side_effect = [
+            (resp_headers, items),
+            (resp_headers, []),
+        ]
+        expected = {
+            'action': 'list_container_part',
+            'container': 'c',
+            'listing': items,
+            'marker': '',
+            'prefix': None,
+            'success': True,
+        }
+        with SwiftService() as swift:
+            list_result_gen = swift.list(container='c', options={
+                'versions': True})
+            self.maxDiff = None
+            for result in list_result_gen:
+                self.assertEqual(result, expected)
+        self.assertEqual(mock_conn.get_container.mock_calls, [
+            mock.call('c', delimiter=None, headers={}, marker='',
+                      prefix=None, query_string='versions=true',
+                      version_marker=''),
+            mock.call('c', delimiter=None, headers={}, marker='test',
+                      prefix=None, query_string='versions=true',
+                      version_marker='1573184903.06720'),
+        ])
+
     @mock.patch('swiftclient.service.get_conn')
     def test_list_queue_size(self, mock_get_conn):
         mock_conn = self._get_mock_connection()
@@ -1042,6 +1181,67 @@ class TestServiceList(_TestServiceBase):
         self.assertEqual(observed_listing, expected_listing)
 
 
+class TestServiceStat(_TestServiceBase):
+
+    maxDiff = None
+
+    @mock.patch('swiftclient.service.Connection')
+    def test_stat_object(self, mock_connection_class):
+        mock_conn = mock_connection_class.return_value
+        mock_conn.url = 'http://saio/v1/AUTH_test'
+        mock_conn.head_object.return_value = {}
+        expected = {
+            'action': 'stat_object',
+            'container': 'c',
+            'object': 'o',
+            'headers': {},
+            'items': [('Account', 'AUTH_test'),
+                      ('Container', 'c'),
+                      ('Object', 'o'),
+                      ('Content Type', None),
+                      ('Content Length', '0'),
+                      ('Last Modified', None),
+                      ('ETag', None),
+                      ('Manifest', None)],
+            'success': True}
+        with SwiftService() as swift:
+            stat_results = swift.stat(container='c', objects='o')
+            for stat_result in stat_results:
+                self.assertEqual(stat_result, expected)
+        self.assertEqual(mock_conn.head_object.mock_calls, [
+            mock.call('c', 'o', headers={}, query_string=None),
+        ])
+
+    @mock.patch('swiftclient.service.Connection')
+    def test_stat_versioned_object(self, mock_connection_class):
+        mock_conn = mock_connection_class.return_value
+        mock_conn.url = 'http://saio/v1/AUTH_test'
+        mock_conn.head_object.return_value = {}
+        expected = {
+            'action': 'stat_object',
+            'container': 'c',
+            'object': 'o',
+            'headers': {},
+            'items': [('Account', 'AUTH_test'),
+                      ('Container', 'c'),
+                      ('Object', 'o'),
+                      ('Content Type', None),
+                      ('Content Length', '0'),
+                      ('Last Modified', None),
+                      ('ETag', None),
+                      ('Manifest', None)],
+            'success': True}
+        with SwiftService() as swift:
+            stat_results = swift.stat(container='c', objects='o', options={
+                'version_id': '234567.8'})
+            for stat_result in stat_results:
+                self.assertEqual(stat_result, expected)
+        self.assertEqual(mock_conn.head_object.mock_calls, [
+            mock.call('c', 'o', headers={},
+                      query_string='version-id=234567.8'),
+        ])
+
+
 class TestService(unittest.TestCase):
 
     def test_upload_with_bad_segment_size(self):
@@ -1791,13 +1991,14 @@ class TestServiceUpload(_TestServiceBase):
             mock_conn.head_object.assert_called_with('test_c', 'test_o')
             expected = [
                 mock.call('test_c_segments', prefix='test_o/prefix',
-                          marker='', delimiter=None, headers={}),
+                          marker='', delimiter=None, headers={},
+                          query_string=None, version_marker=''),
                 mock.call('test_c_segments', prefix='test_o/prefix',
                           marker="test_o/prefix/01", delimiter=None,
-                          headers={}),
+                          headers={}, query_string=None, version_marker=''),
                 mock.call('test_c_segments', prefix='test_o/prefix',
                           marker="test_o/prefix/02", delimiter=None,
-                          headers={}),
+                          headers={}, query_string=None, version_marker=''),
             ]
             mock_conn.get_container.assert_has_calls(expected)
 
@@ -2332,6 +2533,29 @@ class TestServiceDownload(_TestServiceBase):
         self.assertEqual(resp['object'], 'test')
         self.assertEqual(resp['path'], 'test')
 
+    def test_download_version_id(self):
+        self.opts['version_id'] = '23456.7'
+        with mock.patch('swiftclient.service.Connection') as mock_conn:
+            header = {'content-length': self.obj_len,
+                      'etag': self.obj_etag}
+            mock_conn.get_object.return_value = header, self._readbody()
+
+            resp = SwiftService()._download_object_job(mock_conn,
+                                                       'c',
+                                                       'test',
+                                                       self.opts)
+
+        self.assertIsNone(resp.get('error'))
+        self.assertIs(True, resp['success'])
+        self.assertEqual(resp['action'], 'download_object')
+        self.assertEqual(resp['object'], 'test')
+        self.assertEqual(resp['path'], 'test')
+        self.assertEqual(mock_conn.get_object.mock_calls, [
+            mock.call(
+                'c', 'test', headers={}, query_string='version-id=23456.7',
+                resp_chunk_size=65536, response_dict={}),
+        ])
+
     @mock.patch('swiftclient.service.interruptable_as_completed')
     @mock.patch('swiftclient.service.SwiftService._download_container')
     @mock.patch('swiftclient.service.SwiftService._download_object_job')
@@ -2545,17 +2769,17 @@ class TestServiceDownload(_TestServiceBase):
                           delimiter=None,
                           prefix='test_o/prefix',
                           marker='',
-                          headers={}),
+                          headers={}, query_string=None, version_marker=''),
                 mock.call('test_c_segments',
                           delimiter=None,
                           prefix='test_o/prefix',
                           marker='test_o/prefix/2',
-                          headers={}),
+                          headers={}, query_string=None, version_marker=''),
                 mock.call('test_c_segments',
                           delimiter=None,
                           prefix='test_o/prefix',
                           marker='test_o/prefix/3',
-                          headers={})])
+                          headers={}, query_string=None, version_marker='')])
 
     def test_download_object_job_skip_identical_nested_slo(self):
         with tempfile.NamedTemporaryFile() as f:
@@ -2682,6 +2906,7 @@ class TestServiceDownload(_TestServiceBase):
                         obj='test_o',
                         options=options)
 
+            self.maxDiff = None
             self.assertEqual(r, expected_r)
 
             self.assertEqual(mock_conn.get_container.mock_calls, [
@@ -2689,17 +2914,17 @@ class TestServiceDownload(_TestServiceBase):
                           delimiter=None,
                           prefix='test_o/prefix',
                           marker='',
-                          headers={}),
+                          headers={}, query_string=None, version_marker=''),
                 mock.call('test_c_segments',
                           delimiter=None,
                           prefix='test_o/prefix',
                           marker='test_o/prefix/2',
-                          headers={}),
+                          headers={}, query_string=None, version_marker=''),
                 mock.call('test_c_segments',
                           delimiter=None,
                           prefix='test_o/prefix',
                           marker='test_o/prefix/3',
-                          headers={})])
+                          headers={}, query_string=None, version_marker='')])
             self.assertEqual(mock_conn.get_object.mock_calls, [
                 mock.call('test_c',
                           'test_o',

test/unit/test_shell.py

@@ -241,6 +241,30 @@ class TestShell(unittest.TestCase):
         self.assertEqual(connection.return_value.head_container.mock_calls, [
             mock.call('container', headers={'Skip-Middleware': 'Test'})])
 
+    @mock.patch('swiftclient.service.Connection')
+    def test_stat_version_id(self, connection):
+        argv = ["", "stat", "--version-id", "1"]
+        with self.assertRaises(SystemExit) as caught:
+            swiftclient.shell.main(argv)
+        self.assertEqual(str(caught.exception),
+                         "--version-id option only allowed for "
+                         "object stats")
+
+        argv = ["", "stat", "--version-id", "1", "container"]
+        with self.assertRaises(SystemExit) as caught:
+            swiftclient.shell.main(argv)
+        self.assertEqual(str(caught.exception),
+                         "--version-id option only allowed for "
+                         "object stats")
+
+        argv = ["", "stat", "--version-id", "1", "container", "object"]
+        connection.return_value.head_object.return_value = {}
+        with CaptureOutput():
+            swiftclient.shell.main(argv)
+        self.assertEqual([mock.call('container', 'object', headers={},
+                                    query_string='version-id=1')],
+                         connection.return_value.head_object.mock_calls)
+
     @mock.patch('swiftclient.service.Connection')
     def test_stat_object(self, connection):
         return_headers = {
@@ -295,7 +319,45 @@ class TestShell(unittest.TestCase):
                              '      Manifest: manifest\n')
         self.assertEqual(connection.return_value.head_object.mock_calls, [
             mock.call('container', 'object',
-                      headers={'Skip-Middleware': 'Test'})])
+                      headers={'Skip-Middleware': 'Test'},
+                      query_string=None)])
+
+    def test_list_account_with_delimiter(self):
+        argv = ["", "list", "--delimiter", "foo"]
+        with self.assertRaises(SystemExit) as caught:
+            swiftclient.shell.main(argv)
+        self.assertEqual(str(caught.exception),
+                         "-d option only allowed for "
+                         "container listings")
+
+    @mock.patch('swiftclient.service.Connection')
+    def test_list_container_with_versions(self, connection):
+        connection.return_value.get_container.side_effect = [
+            [None, [
+                {'name': 'foo', 'version_id': '2'},
+                {'name': 'foo', 'version_id': '1'},
+            ]],
+            [None, []],
+        ]
+        argv = ["", "list", "container", "--versions"]
+        with CaptureOutput(suppress_systemexit=True) as output:
+            swiftclient.shell.main(argv)
+        calls = [mock.call('container', delimiter=None, headers={}, marker='',
+                           prefix=None, query_string='versions=true',
+                           version_marker=''),
+                 mock.call('container', delimiter=None, headers={},
+                           marker='foo', prefix=None,
+                           query_string='versions=true', version_marker='1')]
+        connection.return_value.get_container.assert_has_calls(calls)
+        self.assertEqual(output.out, 'foo\nfoo\n')
+
+    def test_list_account_with_versions(self):
+        argv = ["", "list", "--versions"]
+        with self.assertRaises(SystemExit) as caught:
+            swiftclient.shell.main(argv)
+        self.assertEqual(str(caught.exception),
+                         "--versions option only allowed for "
+                         "container listings")
 
     @mock.patch('swiftclient.service.Connection')
     def test_list_json(self, connection):
@@ -431,9 +493,11 @@ class TestShell(unittest.TestCase):
             swiftclient.shell.main(argv)
             calls = [
                 mock.call('container', marker='',
-                          delimiter=None, prefix=None, headers={}),
+                          delimiter=None, prefix=None, headers={},
+                          query_string=None, version_marker=''),
                 mock.call('container', marker='object_a',
-                          delimiter=None, prefix=None, headers={})]
+                          delimiter=None, prefix=None, headers={},
+                          query_string=None, version_marker='')]
             connection.return_value.get_container.assert_has_calls(calls)
 
             self.assertEqual(output.out, 'object_a\n')
@@ -450,9 +514,11 @@ class TestShell(unittest.TestCase):
             swiftclient.shell.main(argv)
             calls = [
                 mock.call('container', marker='',
-                          delimiter=None, prefix=None, headers={}),
+                          delimiter=None, prefix=None, headers={},
+                          query_string=None, version_marker=''),
                 mock.call('container', marker='object_a',
-                          delimiter=None, prefix=None, headers={})]
+                          delimiter=None, prefix=None, headers={},
+                          query_string=None, version_marker='')]
             connection.return_value.get_container.assert_has_calls(calls)
 
             self.assertEqual(output.out,
@@ -472,14 +538,44 @@ class TestShell(unittest.TestCase):
             calls = [
                 mock.call('container', marker='',
                           delimiter=None, prefix=None,
-                          headers={'Skip-Middleware': 'Test'}),
+                          headers={'Skip-Middleware': 'Test'},
+                          query_string=None, version_marker=''),
                 mock.call('container', marker='object_a',
                           delimiter=None, prefix=None,
-                          headers={'Skip-Middleware': 'Test'})]
+                          headers={'Skip-Middleware': 'Test'},
+                          query_string=None, version_marker='')]
             connection.return_value.get_container.assert_has_calls(calls)
 
             self.assertEqual(output.out, 'object_a\n')
 
+    @mock.patch('swiftclient.service.Connection')
+    def test_download_version_id(self, connection):
+        argv = ["", "download", "--yes-all", "--version-id", "5"]
+        with self.assertRaises(SystemExit) as caught:
+            swiftclient.shell.main(argv)
+        self.assertEqual(str(caught.exception),
+                         "--version-id option only allowed for "
+                         "object downloads")
+
+        argv = ["", "download", "--version-id", "2", "container"]
+        with self.assertRaises(SystemExit) as caught:
+            swiftclient.shell.main(argv)
+        self.assertEqual(str(caught.exception),
+                         "--version-id option only allowed for "
+                         "object downloads")
+
+        argv = ["", "download", "--version-id", "1", "container", "object"]
+        connection.return_value.head_object.return_value = {}
+        connection.return_value.get_object.return_value = {}, ''
+        connection.return_value.attempts = 0
+        with CaptureOutput():
+            swiftclient.shell.main(argv)
+        self.assertEqual([mock.call('container', 'object', headers={},
+                                    query_string='version-id=1',
+                                    resp_chunk_size=65536,
+                                    response_dict={})],
+                         connection.return_value.get_object.mock_calls)
+
     @mock.patch('swiftclient.service.makedirs')
     @mock.patch('swiftclient.service.Connection')
     def test_download(self, connection, makedirs):
@@ -1085,6 +1181,33 @@ class TestShell(unittest.TestCase):
         check_good(["--object-threads", "1"])
         check_good(["--container-threads", "1"])
 
+    @mock.patch('swiftclient.service.Connection')
+    def test_delete_version_id(self, connection):
+        argv = ["", "delete", "--yes-all", "--version-id", "3"]
+        with self.assertRaises(SystemExit) as caught:
+            swiftclient.shell.main(argv)
+        self.assertEqual(str(caught.exception),
+                         "--version-id option only allowed for "
+                         "object deletes")
+
+        argv = ["", "delete", "--version-id", "1", "container"]
+        with self.assertRaises(SystemExit) as caught:
+            swiftclient.shell.main(argv)
+        self.assertEqual(str(caught.exception),
+                         "--version-id option only allowed for "
+                         "object deletes")
+
+        argv = ["", "delete", "--version-id", "1", "container", "object"]
+        connection.return_value.head_object.return_value = {}
+        connection.return_value.delete_object.return_value = None
+        connection.return_value.attempts = 0
+        with CaptureOutput():
+            swiftclient.shell.main(argv)
+        self.assertEqual([mock.call('container', 'object', headers={},
+                                    query_string='version-id=1',
+                                    response_dict={})],
+                         connection.return_value.delete_object.mock_calls)
+
     @mock.patch.object(swiftclient.service.SwiftService,
                        '_bulk_delete_page_size', lambda *a: 1)
     @mock.patch('swiftclient.service.Connection')
@@ -1094,10 +1217,12 @@ class TestShell(unittest.TestCase):
             [None, [{'name': 'empty_container'}]],
             [None, []],
         ]
+        # N.B: missing --versions flag, version-id gets ignored
+        # only latest object is deleted
         connection.return_value.get_container.side_effect = [
             [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}]],
             [None, []],
-            [None, [{'name': 'object'}]],
+            [None, [{'name': 'object', 'version_id': 1}]],
             [None, []],
             [None, []],
         ]
@@ -1107,11 +1232,48 @@ class TestShell(unittest.TestCase):
         connection.return_value.delete_object.return_value = None
         swiftclient.shell.main(argv)
         connection.return_value.delete_object.assert_has_calls([
-            mock.call('container', 'object', query_string=None,
+            mock.call('container', 'object', query_string='',
                       response_dict={}, headers={}),
-            mock.call('container', 'obj\xe9ct2', query_string=None,
+            mock.call('container', 'obj\xe9ct2', query_string='',
                       response_dict={}, headers={}),
-            mock.call('container2', 'object', query_string=None,
+            mock.call('container2', 'object', query_string='',
+                      response_dict={}, headers={})], any_order=True)
+        self.assertEqual(3, connection.return_value.delete_object.call_count,
+                         'Expected 3 calls but found\n%r'
+                         % connection.return_value.delete_object.mock_calls)
+        self.assertEqual(
+            connection.return_value.delete_container.mock_calls, [
+                mock.call('container', response_dict={}, headers={}),
+                mock.call('container2', response_dict={}, headers={}),
+                mock.call('empty_container', response_dict={}, headers={})])
+
+    @mock.patch.object(swiftclient.service.SwiftService,
+                       '_bulk_delete_page_size', lambda *a: 1)
+    @mock.patch('swiftclient.service.Connection')
+    def test_delete_account_versions(self, connection):
+        connection.return_value.get_account.side_effect = [
+            [None, [{'name': 'container'}, {'name': 'container2'}]],
+            [None, [{'name': 'empty_container'}]],
+            [None, []],
+        ]
+        connection.return_value.get_container.side_effect = [
+            [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}]],
+            [None, []],
+            [None, [{'name': 'obj', 'version_id': 1}]],
+            [None, []],
+            [None, []],
+        ]
+        connection.return_value.attempts = 0
+        argv = ["", "delete", "--all", "--versions"]
+        connection.return_value.head_object.return_value = {}
+        connection.return_value.delete_object.return_value = None
+        swiftclient.shell.main(argv)
+        connection.return_value.delete_object.assert_has_calls([
+            mock.call('container', 'object', query_string='',
+                      response_dict={}, headers={}),
+            mock.call('container', 'obj\xe9ct2', query_string='',
+                      response_dict={}, headers={}),
+            mock.call('container2', 'obj', query_string='version-id=1',
                       response_dict={}, headers={})], any_order=True)
         self.assertEqual(3, connection.return_value.delete_object.call_count,
                          'Expected 3 calls but found\n%r'
@@ -1323,9 +1485,39 @@ class TestShell(unittest.TestCase):
         connection.return_value.delete_container.assert_called_with(
             'container', response_dict={}, headers={})
         connection.return_value.delete_object.assert_called_with(
-            'container', 'object', query_string=None, response_dict={},
+            'container', 'object', query_string='', response_dict={},
             headers={})
 
+    @mock.patch.object(swiftclient.service.SwiftService,
+                       '_bulk_delete_page_size', lambda *a: 1)
+    @mock.patch('swiftclient.service.Connection')
+    def test_delete_container_versions(self, connection):
+        argv = ["", "delete", "--versions", "container", "obj"]
+        with self.assertRaises(SystemExit) as caught:
+            swiftclient.shell.main(argv)
+        self.assertEqual(str(caught.exception),
+                         "--versions option not allowed for object deletes")
+
+        connection.return_value.get_container.side_effect = [
+            [None, [{'name': 'object', 'version_id': 2},
+                    {'name': 'object', 'version_id': 1}]],
+            [None, []],
+        ]
+        connection.return_value.attempts = 0
+        argv = ["", "delete", "--versions", "container"]
+        connection.return_value.head_object.return_value = {}
+        swiftclient.shell.main(argv)
+        connection.return_value.delete_container.assert_called_with(
+            'container', response_dict={}, headers={})
+        expected_calls = [
+            mock.call('container', 'object', query_string='version-id=2',
+                      response_dict={}, headers={}),
+            mock.call('container', 'object', query_string='version-id=1',
+                      response_dict={}, headers={})]
+
+        self.assertEqual(connection.return_value.delete_object.mock_calls,
+                         expected_calls)
+
     @mock.patch.object(swiftclient.service.SwiftService,
                        '_bulk_delete_page_size', lambda *a: 1)
     @mock.patch('swiftclient.service.Connection')
@@ -1342,7 +1534,7 @@ class TestShell(unittest.TestCase):
             'container', response_dict={},
             headers={'Skip-Middleware': 'Test'})
         connection.return_value.delete_object.assert_called_with(
-            'container', 'object', query_string=None, response_dict={},
+            'container', 'object', query_string='', response_dict={},
             headers={'Skip-Middleware': 'Test'})
 
     @mock.patch.object(swiftclient.service.SwiftService,
@@ -1408,7 +1600,7 @@ class TestShell(unittest.TestCase):
         connection.return_value.attempts = 0
         swiftclient.shell.main(argv)
         connection.return_value.delete_object.assert_called_with(
-            'container', 'object', query_string=None, response_dict={},
+            'container', 'object', query_string='', response_dict={},
             headers={})
 
     @mock.patch.object(swiftclient.service.SwiftService,