From 4a2465fb12ff7287b62b6941fb8ae43e100adc25 Mon Sep 17 00:00:00 2001 From: Marek Kaleta Date: Mon, 15 Feb 2016 12:14:17 +0100 Subject: [PATCH] Add copy object method Implement copy object method in swiftclient Connection, Service and CLI. Although COPY functionality can be accomplished with 'X-Copy-From' header in PUT request, using copy is more convenient especially when using copy for updating object metadata non-destructively. Closes-Bug: 1474939 Change-Id: I1338ac411f418f4adb3d06753d044a484a7f32a4 --- doc/manpages/swift.1 | 13 ++ doc/source/cli.rst | 14 ++ doc/source/service-api.rst | 82 +++++++++- examples/copy.py | 30 ++++ swiftclient/client.py | 71 +++++++++ swiftclient/service.py | 225 ++++++++++++++++++++++++++- swiftclient/shell.py | 102 +++++++++++- tests/functional/test_swiftclient.py | 40 +++++ tests/unit/test_service.py | 106 +++++++++++++ tests/unit/test_shell.py | 133 ++++++++++++++++ tests/unit/test_swiftclient.py | 104 +++++++++++++ 11 files changed, 914 insertions(+), 6 deletions(-) create mode 100644 examples/copy.py diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index b9f99c4d..7c2ee896 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -79,6 +79,19 @@ For more details and options see swift post \-\-help. \fBExample\fR: post \-m Color:Blue \-m Size:Large .RE +\fBcopy\fR [\fIcommand-options\fR] \fIcontainer\fR \fIobject\fR +.RS 4 +Copies an object to a new destination or adds user metadata to the object (current +user metadata will be preserved, in contrast with the post command) depending +on the args given. The \-\-destination option sets the destination in the form +/container/object. If not set, the object will be copied onto itself which is useful +for adding metadata. The \-M or \-\-fresh\-metadata option copies the object without +the existing user metadata. The \-m or \-\-meta option is always allowed and is used +to define the user metadata items to set in the form Name:Value (this option +can be repeated). +For more details and options see swift copy \-\-help. +.RE + \fBdownload\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] [\fIobject\fR] [...] .RS 4 Downloads everything in the account (with \-\-all), or everything in a diff --git a/doc/source/cli.rst b/doc/source/cli.rst index 12de02ff..76630be9 100644 --- a/doc/source/cli.rst +++ b/doc/source/cli.rst @@ -199,6 +199,20 @@ Delete of manifest objects will be deleted as well, unless you specify the ``--leave-segments`` option. +Copy +---- + + ``copy [command-options] container object`` + + Copies an object to a new destination or adds user metadata to an object. Depending + on the options supplied, you can preserve existing metadata in contrast to the post + command. The ``--destination`` option sets the copy target destination in the form + ``/container/object``. If not set, the object will be copied onto itself which is useful + for adding metadata. You can use the ``-M`` or ``--fresh-metadata`` option to copy + an object without existing user meta data, and the ``-m`` or ``--meta`` option + to define user meta data items to set in the form ``Name:Value``. You can repeat + this option. For example: ``copy -m Color:Blue -m Size:Large``. + Capabilities ------------ diff --git a/doc/source/service-api.rst b/doc/source/service-api.rst index 50dd80e8..aeb3daac 100644 --- a/doc/source/service-api.rst +++ b/doc/source/service-api.rst @@ -217,6 +217,17 @@ Options are downloaded in lexically-sorted order. Setting this option to ``True`` gives the same shuffling behaviour as the CLI. + ``destination``: ``None`` + When copying objects, this specifies the destination where the object + will be copied to. The default of None means copy will be the same as + source. + + ``fresh_metadata``: ``None`` + When copying objects, this specifies that the object metadata on the + source will *not* be applied to the destination object - the + destination object will have a new fresh set of metadata that includes + *only* the metadata specified in the meta option if any at all. + Other available options can be found in ``swiftclient/service.py`` in the source code for ``python-swiftclient``. Each ``SwiftService`` method also allows for an optional dictionary to override those specified at init time, and the @@ -735,13 +746,76 @@ Example The code below demonstrates the use of ``delete`` to remove a given list of objects from a specified container. As the objects are deleted the transaction -id of the relevant request is printed along with the object name and number -of attempts required. By printing the transaction id, the printed operations +ID of the relevant request is printed along with the object name and number +of attempts required. By printing the transaction ID, the printed operations can be easily linked to events in the swift server logs: .. literalinclude:: ../../examples/delete.py :language: python +Copy +~~~~ + +Copy can be called to copy an object or update the metadata on the given items. + +Each element of the object list may be a plain string of the object name, or a +``SwiftCopyObject`` that allows finer control over the options applied to each +of the individual copy operations (destination, fresh_metadata, options). + +Destination should be in format /container/object; if not set, the object will be +copied onto itself. Fresh_metadata sets mode of operation on metadata. If not set, +current object user metadata will be copied/preserved; if set, all current user +metadata will be removed. + +Returns an iterator over the results generated for each object copy (and may +also include the results of creating destination containers). + +When a string is given for the object name, destination and fresh metadata will +default to None and None, which result in adding metadata to existing objects. + +Successful copy results are dictionaries as described below: + +.. code-block:: python + + { + 'action': 'copy_object', + 'success': True, + 'container': , + 'object': , + 'destination': , + 'headers': {}, + 'fresh_metadata': , + 'response_dict': + } + +Any failure in a copy operation will return a failure dictionary as described +below: + +.. code-block:: python + + { + 'action': 'copy_object', + 'success': False, + 'container': , + 'object': , + 'destination': , + 'headers': {}, + 'fresh_metadata': , + 'response_dict': , + 'error': , + 'traceback': , + 'error_timestamp': + } + +Example +------- + +The code below demonstrates the use of ``copy`` to add new user metadata for +objects a and b, and to copy object c to d (with added metadata). + +.. literalinclude:: ../../examples/copy.py + :language: python + Capabilities ~~~~~~~~~~~~ @@ -764,7 +838,7 @@ returned with the contents described below: } The contents of the capabilities dictionary contain the core swift capabilities -under the key ``swift``, all other keys show the configuration options for +under the key ``swift``; all other keys show the configuration options for additional middlewares deployed in the proxy pipeline. An example capabilities dictionary is given below: @@ -819,7 +893,7 @@ Example ^^^^^^^ The code below demonstrates the use of ``capabilities`` to determine if the -Swift cluster supports static large objects, and if so, the maximum number of +Swift cluster supports static large objects, and if so, the maximum number of segments that can be described in a single manifest file, along with the size restrictions on those objects: diff --git a/examples/copy.py b/examples/copy.py new file mode 100644 index 00000000..e928db4e --- /dev/null +++ b/examples/copy.py @@ -0,0 +1,30 @@ +import logging + +from swiftclient.service import SwiftService, SwiftCopyObject, SwiftError + +logging.basicConfig(level=logging.ERROR) +logging.getLogger("requests").setLevel(logging.CRITICAL) +logging.getLogger("swiftclient").setLevel(logging.CRITICAL) +logger = logging.getLogger(__name__) + +with SwiftService() as swift: + try: + obj = SwiftCopyObject("c", {"Destination": "/cont/d"}) + for i in swift.copy( + "cont", ["a", "b", obj], + {"meta": ["foo:bar"], "Destination": "/cc"}): + if i["success"]: + if i["action"] == "copy_object": + print( + "object %s copied from /%s/%s" % + (i["destination"], i["container"], i["object"]) + ) + if i["action"] == "create_container": + print( + "container %s created" % i["container"] + ) + else: + if "error" in i and isinstance(i["error"], Exception): + raise i["error"] + except SwiftError as e: + logger.error(e.value) diff --git a/swiftclient/client.py b/swiftclient/client.py index 744a876b..5b3fa725 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1319,6 +1319,70 @@ def post_object(url, token, container, name, headers, http_conn=None, raise ClientException.from_response(resp, 'Object POST failed', body) +def copy_object(url, token, container, name, destination=None, + headers=None, fresh_metadata=None, http_conn=None, + response_dict=None, service_token=None): + """ + Copy object + + :param url: storage URL + :param token: auth token; if None, no token will be sent + :param container: container name that the source object is in + :param name: source object name + :param destination: The container and object name of the destination object + in the form of /container/object; if None, the copy + will use the source as the destination. + :param headers: additional headers to include in the request + :param fresh_metadata: Enables object creation that omits existing user + metadata, default None + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :param response_dict: an optional dictionary into which to place + the response - status, reason and headers + :param service_token: service auth token + :raises ClientException: HTTP COPY request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + + path = parsed.path + container = quote(container) + name = quote(name) + path = '%s/%s/%s' % (path.rstrip('/'), container, name) + + headers = dict(headers) if headers else {} + + if destination is not None: + headers['Destination'] = quote(destination) + elif container and name: + headers['Destination'] = '/%s/%s' % (container, name) + + if token is not None: + headers['X-Auth-Token'] = token + if service_token is not None: + headers['X-Service-Token'] = service_token + + if fresh_metadata is not None: + # remove potential fresh metadata headers + for fresh_hdr in [hdr for hdr in headers.keys() + if hdr.lower() == 'x-fresh-metadata']: + headers.pop(fresh_hdr) + headers['X-Fresh-Metadata'] = 'true' if fresh_metadata else 'false' + + conn.request('COPY', path, '', headers) + resp = conn.getresponse() + body = resp.read() + http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'COPY',), + {'headers': headers}, resp, body) + + store_response(resp, response_dict) + + if resp.status < 200 or resp.status >= 300: + raise ClientException.from_response(resp, 'Object COPY failed', body) + + def delete_object(url, token=None, container=None, name=None, http_conn=None, headers=None, proxy=None, query_string=None, response_dict=None, service_token=None): @@ -1711,6 +1775,13 @@ class Connection(object): return self._retry(None, post_object, container, obj, headers, response_dict=response_dict) + def copy_object(self, container, obj, destination=None, headers=None, + fresh_metadata=None, response_dict=None): + """Wrapper for :func:`copy_object`""" + return self._retry(None, copy_object, container, obj, destination, + headers, fresh_metadata, + response_dict=response_dict) + def delete_object(self, container, obj, query_string=None, response_dict=None): """Wrapper for :func:`delete_object`""" diff --git a/swiftclient/service.py b/swiftclient/service.py index 12d3f210..a813d163 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -200,7 +200,9 @@ _default_local_options = { 'human': False, 'dir_marker': False, 'checksum': True, - 'shuffle': False + 'shuffle': False, + 'destination': None, + 'fresh_metadata': False, } POLICY = 'X-Storage-Policy' @@ -330,6 +332,42 @@ class SwiftPostObject(object): self.options = options +class SwiftCopyObject(object): + """ + Class for specifying an object copy, + allowing the destination/headers/metadata/fresh_metadata to be specified + separately for each individual object. + destination and fresh_metadata should be set in options + """ + def __init__(self, object_name, options=None): + if not isinstance(object_name, string_types) or not object_name: + raise SwiftError( + "Object names must be specified as non-empty strings" + ) + + self.object_name = object_name + self.options = options + + if self.options is None: + self.destination = None + self.fresh_metadata = False + else: + self.destination = self.options.get('destination') + self.fresh_metadata = self.options.get('fresh_metadata', False) + + if self.destination is not None: + destination_components = self.destination.split('/') + if destination_components[0] or len(destination_components) < 2: + raise SwiftError("destination must be in format /cont[/obj]") + if not destination_components[-1]: + raise SwiftError("destination must not end in a slash") + if len(destination_components) == 2: + # only container set in destination + self.destination = "{0}/{1}".format( + self.destination, object_name + ) + + class _SwiftReader(object): """ Class for downloading objects from swift and raising appropriate @@ -2389,6 +2427,191 @@ class SwiftService(object): return res + # Copy related methods + # + def copy(self, container, objects, options=None): + """ + Copy operations on a list of objects in a container. Destination + containers will be created. + + :param container: The container from which to copy the objects. + :param objects: A list of object names (strings) or SwiftCopyObject + instances containing an object name and an + options dict (can be None) to override the options for + that individual copy operation:: + + [ + 'object_name', + SwiftCopyObject( + 'object_name', + options={ + 'destination': '/container/object', + 'fresh_metadata': False, + ... + }), + ... + ] + + The options dict is described below. + :param options: A dictionary containing options to override the global + options specified during the service object creation. + These options are applied to all copy operations + performed by this call, unless overridden on a per + object basis. + The options "destination" and "fresh_metadata" do + not need to be set, in this case objects will be + copied onto themselves and metadata will not be + refreshed. + The option "destination" can also be specified in the + format '/container', in which case objects without an + explicit destination will be copied to the destination + /container/original_object_name. Combinations of + multiple objects and a destination in the format + '/container/object' is invalid. Possible options are + given below:: + + { + 'meta': [], + 'header': [], + 'destination': '/container/object', + 'fresh_metadata': False, + } + + :returns: A generator returning the results of copying the given list + of objects. + + :raises: SwiftError + """ + if options is not None: + options = dict(self._options, **options) + else: + options = self._options + + # Try to create the container, just in case it doesn't exist. If this + # fails, it might just be because the user doesn't have container PUT + # permissions, so we'll ignore any error. If there's really a problem, + # it'll surface on the first object COPY. + containers = set( + next(p for p in obj.destination.split("/") if p) + for obj in objects + if isinstance(obj, SwiftCopyObject) and obj.destination + ) + if options.get('destination'): + destination_split = options['destination'].split('/') + if destination_split[0]: + raise SwiftError("destination must be in format /cont[/obj]") + _str_objs = [ + o for o in objects if not isinstance(o, SwiftCopyObject) + ] + if len(destination_split) > 2 and len(_str_objs) > 1: + # TODO (clayg): could be useful to copy multiple objects into + # a destination like "/container/common/prefix/for/objects/" + # where the trailing "/" indicates the destination option is a + # prefix! + raise SwiftError("Combination of multiple objects and " + "destination including object is invalid") + if destination_split[-1] == '': + # N.B. this protects the above case + raise SwiftError("destination can not end in a slash") + containers.add(destination_split[1]) + + policy_header = {} + _header = split_headers(options["header"]) + if POLICY in _header: + policy_header[POLICY] = _header[POLICY] + create_containers = [ + self.thread_manager.container_pool.submit( + self._create_container_job, cont, headers=policy_header) + for cont in containers + ] + + # wait for container creation jobs to complete before any COPY + for r in interruptable_as_completed(create_containers): + res = r.result() + yield res + + copy_futures = [] + copy_objects = self._make_copy_objects(objects, options) + for copy_object in copy_objects: + obj = copy_object.object_name + obj_options = copy_object.options + destination = copy_object.destination + fresh_metadata = copy_object.fresh_metadata + headers = split_headers( + options['meta'], 'X-Object-Meta-') + # add header options to the headers object for the request. + headers.update( + split_headers(options['header'], '')) + if obj_options is not None: + if 'meta' in obj_options: + headers.update( + split_headers( + obj_options['meta'], 'X-Object-Meta-' + ) + ) + if 'header' in obj_options: + headers.update( + split_headers(obj_options['header'], '') + ) + + copy = self.thread_manager.object_uu_pool.submit( + self._copy_object_job, container, obj, destination, + headers, fresh_metadata + ) + copy_futures.append(copy) + + for r in interruptable_as_completed(copy_futures): + res = r.result() + yield res + + @staticmethod + def _make_copy_objects(objects, options): + copy_objects = [] + + for o in objects: + if isinstance(o, string_types): + obj = SwiftCopyObject(o, options) + copy_objects.append(obj) + elif isinstance(o, SwiftCopyObject): + copy_objects.append(o) + else: + raise SwiftError( + "The copy operation takes only strings or " + "SwiftCopyObjects as input", + obj=o) + + return copy_objects + + @staticmethod + def _copy_object_job(conn, container, obj, destination, headers, + fresh_metadata): + response_dict = {} + res = { + 'success': True, + 'action': 'copy_object', + 'container': container, + 'object': obj, + 'destination': destination, + 'headers': headers, + 'fresh_metadata': fresh_metadata, + 'response_dict': response_dict + } + try: + conn.copy_object( + container, obj, destination=destination, headers=headers, + fresh_metadata=fresh_metadata, response_dict=response_dict) + except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) + res.update({ + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + }) + + return res + # Capabilities related methods # def capabilities(self, url=None, refresh_cache=False): diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 5eafe0b6..8da0e7ba 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -46,7 +46,7 @@ except ImportError: from pipes import quote as sh_quote BASENAME = 'swift' -commands = ('delete', 'download', 'list', 'post', 'stat', 'upload', +commands = ('delete', 'download', 'list', 'post', 'copy', 'stat', 'upload', 'capabilities', 'info', 'tempurl', 'auth') @@ -746,6 +746,105 @@ def st_post(parser, args, output_manager): output_manager.error(e.value) +st_copy_options = '''[--destination ] [--fresh-metadata] + [--meta ] [--header
] container object +''' + +st_copy_help = ''' +Copies object to new destination, optionally updates objects metadata. +If destination is not set, will update metadata of object + +Positional arguments: + container Name of container to copy from. + object Name of object to copy. Specify multiple times + for multiple objects + +Optional arguments: + -d, --destination + The container and name of the destination object. Name + of destination object can be ommited, then will be + same as name of source object. Supplying multiple + objects and destination with object name is invalid. + -M, --fresh-metadata Copy the object without any existing metadata, + If not set, metadata will be preserved or appended + -m, --meta + Sets a meta data item. This option may be repeated. + Example: -m Color:Blue -m Size:Large + -H, --header + Adds a customized request header. + This option may be repeated. Example + -H "content-type:text/plain" -H "Content-Length: 4000" +'''.strip('\n') + + +def st_copy(parser, args, output_manager): + parser.add_argument( + '-d', '--destination', help='The container and name of the ' + 'destination object') + parser.add_argument( + '-M', '--fresh-metadata', action='store_true', + help='Copy the object without any existing metadata', default=False) + parser.add_argument( + '-m', '--meta', action='append', dest='meta', default=[], + help='Sets a meta data item. This option may be repeated. ' + 'Example: -m Color:Blue -m Size:Large') + parser.add_argument( + '-H', '--header', action='append', dest='header', + default=[], help='Adds a customized request header. ' + 'This option may be repeated. ' + 'Example: -H "content-type:text/plain" ' + '-H "Content-Length: 4000"') + (options, args) = parse_args(parser, args) + args = args[1:] + + with SwiftService(options=options) as swift: + try: + if len(args) >= 2: + container = args[0] + if '/' in container: + output_manager.error( + 'WARNING: / in container name; you might have ' + "meant '%s' instead of '%s'." % + (args[0].replace('/', ' ', 1), args[0])) + return + objects = [arg for arg in args[1:]] + + for r in swift.copy( + container=container, objects=objects, + options=options): + if r['success']: + if options['verbose']: + if r['action'] == 'copy_object': + output_manager.print_msg( + '%s/%s copied to %s' % ( + r['container'], + r['object'], + r['destination'] or '')) + if r['action'] == 'create_container': + output_manager.print_msg( + 'created container %s' % r['container'] + ) + else: + error = r['error'] + if 'action' in r and r['action'] == 'create_container': + # it is not an error to be unable to create the + # container so print a warning and carry on + output_manager.warning( + 'Warning: failed to create container ' + "'%s': %s", container, error + ) + else: + output_manager.error("%s" % error) + else: + output_manager.error( + 'Usage: %s copy %s\n%s', BASENAME, + st_copy_options, st_copy_help) + return + + except SwiftError as e: + output_manager.error(e.value) + + st_upload_options = '''[--changed] [--skip-identical] [--segment-size ] [--segment-container ] [--leave-segments] [--object-threads ] [--segment-threads ] @@ -1268,6 +1367,7 @@ Positional arguments: for a container. post Updates meta information for the account, container, or object; creates containers if not present. + copy Copies object, optionally adds meta stat Displays information for the account, container, or object. upload Uploads files or directories to the given container. diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index 7a77c071..6e19abdb 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -405,6 +405,46 @@ class TestFunctional(unittest.TestCase): headers = self.conn.head_object(self.containername, self.objectname) self.assertEqual('Something', headers.get('x-object-meta-color')) + def test_copy_object(self): + self.conn.put_object( + self.containername, self.objectname, self.test_data) + self.conn.copy_object(self.containername, + self.objectname, + headers={'x-object-meta-color': 'Something'}) + + headers = self.conn.head_object(self.containername, self.objectname) + self.assertEqual('Something', headers.get('x-object-meta-color')) + + self.conn.copy_object(self.containername, + self.objectname, + headers={'x-object-meta-taste': 'Second'}) + + headers = self.conn.head_object(self.containername, self.objectname) + self.assertEqual('Something', headers.get('x-object-meta-color')) + self.assertEqual('Second', headers.get('x-object-meta-taste')) + + destination = "/%s/%s" % (self.containername, self.objectname_2) + self.conn.copy_object(self.containername, + self.objectname, + destination, + headers={'x-object-meta-taste': 'Second'}) + headers, data = self.conn.get_object(self.containername, + self.objectname_2) + self.assertEqual(self.test_data, data) + self.assertEqual('Something', headers.get('x-object-meta-color')) + self.assertEqual('Second', headers.get('x-object-meta-taste')) + + destination = "/%s/%s" % (self.containername, self.objectname_2) + self.conn.copy_object(self.containername, + self.objectname, + destination, + headers={'x-object-meta-color': 'Else'}, + fresh_metadata=True) + + headers = self.conn.head_object(self.containername, self.objectname_2) + self.assertEqual('Else', headers.get('x-object-meta-color')) + self.assertIsNone(headers.get('x-object-meta-taste')) + def test_get_capabilities(self): resp = self.conn.get_capabilities() self.assertTrue(resp.get('swift')) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index cce8c7a9..5b04c44a 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -69,6 +69,42 @@ class TestSwiftPostObject(unittest.TestCase): self.assertRaises(SwiftError, self.spo, 1) +class TestSwiftCopyObject(unittest.TestCase): + + def setUp(self): + super(TestSwiftCopyObject, self).setUp() + self.sco = swiftclient.service.SwiftCopyObject + + def test_create(self): + sco = self.sco('obj_name') + + self.assertEqual(sco.object_name, 'obj_name') + self.assertIsNone(sco.destination) + self.assertFalse(sco.fresh_metadata) + + sco = self.sco('obj_name', + {'destination': '/dest', 'fresh_metadata': True}) + + self.assertEqual(sco.object_name, 'obj_name') + self.assertEqual(sco.destination, '/dest/obj_name') + self.assertTrue(sco.fresh_metadata) + + sco = self.sco('obj_name', + {'destination': '/dest/new_obj/a', + 'fresh_metadata': False}) + + self.assertEqual(sco.object_name, 'obj_name') + self.assertEqual(sco.destination, '/dest/new_obj/a') + self.assertFalse(sco.fresh_metadata) + + def test_create_with_invalid_name(self): + # empty strings are not allowed as names + self.assertRaises(SwiftError, self.sco, '') + + # names cannot be anything but strings + self.assertRaises(SwiftError, self.sco, 1) + + class TestSwiftReader(unittest.TestCase): def setUp(self): @@ -2344,3 +2380,73 @@ class TestServicePost(_TestServiceBase): res_iter.assert_called_with( [tm_instance.object_uu_pool.submit()] * len(calls)) + + +class TestServiceCopy(_TestServiceBase): + + def setUp(self): + super(TestServiceCopy, self).setUp() + self.opts = swiftclient.service._default_local_options.copy() + + @mock.patch('swiftclient.service.MultiThreadingManager') + @mock.patch('swiftclient.service.interruptable_as_completed') + def test_object_copy(self, inter_compl, thread_manager): + """ + Check copy method translates strings and objects to _copy_object_job + calls correctly + """ + tm_instance = Mock() + thread_manager.return_value = tm_instance + + self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]}) + sco = swiftclient.service.SwiftCopyObject( + "test_sco", + options={'meta': ["meta1:test2"], "header": ["hdr1:test2"], + 'destination': "/cont_new/test_sco"}) + + res = SwiftService().copy('test_c', ['test_o', sco], self.opts) + res = list(res) + + calls = [ + mock.call( + SwiftService._create_container_job, 'cont_new', headers={}), + ] + tm_instance.container_pool.submit.assert_has_calls(calls, + any_order=True) + self.assertEqual( + tm_instance.container_pool.submit.call_count, len(calls)) + + calls = [ + mock.call( + SwiftService._copy_object_job, 'test_c', 'test_o', + None, + { + "X-Object-Meta-Meta1": "test1", + "Hdr1": "test1"}, + False), + mock.call( + SwiftService._copy_object_job, 'test_c', 'test_sco', + '/cont_new/test_sco', + { + "X-Object-Meta-Meta1": "test2", + "Hdr1": "test2"}, + False), + ] + tm_instance.object_uu_pool.submit.assert_has_calls(calls) + self.assertEqual( + tm_instance.object_uu_pool.submit.call_count, len(calls)) + + inter_compl.assert_called_with( + [tm_instance.object_uu_pool.submit()] * len(calls)) + + def test_object_copy_fail_dest(self): + """ + Destination in incorrect format and destination with object + used when multiple objects are copied raises SwiftError + """ + with self.assertRaises(SwiftError): + list(SwiftService().copy('test_c', ['test_o'], + {'destination': 'cont'})) + with self.assertRaises(SwiftError): + list(SwiftService().copy('test_c', ['test_o', 'test_o2'], + {'destination': '/cont/obj'})) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index d82def62..407076b3 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1244,6 +1244,139 @@ class TestShell(unittest.TestCase): self.assertTrue(output.err != '') self.assertTrue(output.err.startswith('Usage')) + @mock.patch('swiftclient.service.Connection') + def test_copy_object_no_destination(self, connection): + argv = ["", "copy", "container", "object", + "--meta", "Color:Blue", + "--header", "content-type:text/plain" + ] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + connection.return_value.copy_object.assert_called_with( + 'container', 'object', destination=None, fresh_metadata=False, + headers={ + 'Content-Type': 'text/plain', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) + self.assertEqual(output.out, 'container/object copied to \n') + + @mock.patch('swiftclient.service.Connection') + def test_copy_object(self, connection): + argv = ["", "copy", "container", "object", + "--meta", "Color:Blue", + "--header", "content-type:text/plain", + "--destination", "/c/o" + ] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + connection.return_value.copy_object.assert_called_with( + 'container', 'object', destination="/c/o", + fresh_metadata=False, + headers={ + 'Content-Type': 'text/plain', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) + self.assertEqual( + output.out, + 'created container c\ncontainer/object copied to /c/o\n' + ) + + @mock.patch('swiftclient.service.Connection') + def test_copy_object_fresh_metadata(self, connection): + argv = ["", "copy", "container", "object", + "--meta", "Color:Blue", "--fresh-metadata", + "--header", "content-type:text/plain", + "--destination", "/c/o" + ] + swiftclient.shell.main(argv) + connection.return_value.copy_object.assert_called_with( + 'container', 'object', destination="/c/o", fresh_metadata=True, + headers={ + 'Content-Type': 'text/plain', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) + + @mock.patch('swiftclient.service.Connection') + def test_copy_two_objects(self, connection): + argv = ["", "copy", "container", "object", "object2", + "--meta", "Color:Blue"] + connection.return_value.copy_object.return_value = None + swiftclient.shell.main(argv) + calls = [ + mock.call( + 'container', 'object', destination=None, + fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, + response_dict={}), + mock.call( + 'container', 'object2', destination=None, + fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, + response_dict={}) + ] + for call in calls: + self.assertIn(call, connection.return_value.copy_object.mock_calls) + self.assertEqual(len(connection.return_value.copy_object.mock_calls), + len(calls)) + + @mock.patch('swiftclient.service.Connection') + def test_copy_two_objects_destination(self, connection): + argv = ["", "copy", "container", "object", "object2", + "--meta", "Color:Blue", "--destination", "/c"] + swiftclient.shell.main(argv) + calls = [ + mock.call( + 'container', 'object', destination="/c/object", + fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, + response_dict={}), + mock.call( + 'container', 'object2', destination="/c/object2", + fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, + response_dict={}) + ] + connection.return_value.copy_object.assert_has_calls(calls) + + @mock.patch('swiftclient.service.Connection') + def test_copy_two_objects_bad_destination(self, connection): + argv = ["", "copy", "container", "object", "object2", + "--meta", "Color:Blue", "--destination", "/c/o"] + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual( + output.err, + 'Combination of multiple objects and destination ' + 'including object is invalid\n') + + @mock.patch('swiftclient.service.Connection') + def test_copy_object_bad_auth(self, connection): + argv = ["", "copy", "container", "object"] + connection.return_value.copy_object.side_effect = \ + swiftclient.ClientException("bad auth") + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, 'bad auth\n') + + def test_copy_object_not_enough_args(self): + argv = ["", "copy", "container"] + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertTrue(output.err != '') + self.assertTrue(output.err.startswith('Usage')) + + def test_copy_bad_container(self): + argv = ["", "copy", "cont/ainer", "object"] + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertTrue(output.err != '') + self.assertTrue(output.err.startswith('WARN')) + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url(self, temp_url): argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index cbb95dbe..cc6a4aca 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -1398,6 +1398,109 @@ class TestPostObject(MockHttpTest): ]) +class TestCopyObject(MockHttpTest): + + def test_server_error(self): + c.http_connection = self.fake_http_connection(500) + self.assertRaises( + c.ClientException, c.copy_object, + 'http://www.test.com/v1/AUTH', 'asdf', 'asdf', 'asdf') + + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + destination='/container2/obj') + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Auth-Token': 'token', + 'Destination': '/container2/obj', + }), + ]) + + def test_service_token(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object('http://www.test.com/v1/AUTH', None, 'container', + 'obj', destination='/container2/obj', + service_token="TOKEN") + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Service-Token': 'TOKEN', + 'Destination': '/container2/obj', + + }), + ]) + + def test_headers(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + destination='/container2/obj', + headers={'some-hdr': 'a', 'other-hdr': 'b'}) + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Auth-Token': 'token', + 'Destination': '/container2/obj', + 'some-hdr': 'a', + 'other-hdr': 'b', + }), + ]) + + def test_fresh_metadata_default(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + '/container2/obj', {'x-fresh-metadata': 'hdr-value'}) + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Auth-Token': 'token', + 'Destination': '/container2/obj', + 'X-Fresh-Metadata': 'hdr-value', + }), + ]) + + def test_fresh_metadata_true(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + destination='/container2/obj', + headers={'x-fresh-metadata': 'hdr-value'}, + fresh_metadata=True) + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Auth-Token': 'token', + 'Destination': '/container2/obj', + 'X-Fresh-Metadata': 'true', + }), + ]) + + def test_fresh_metadata_false(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + destination='/container2/obj', + headers={'x-fresh-metadata': 'hdr-value'}, + fresh_metadata=False) + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'x-auth-token': 'token', + 'Destination': '/container2/obj', + 'X-Fresh-Metadata': 'false', + }), + ]) + + def test_no_destination(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj') + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'x-auth-token': 'token', + 'Destination': '/container/obj', + }), + ]) + + class TestDeleteObject(MockHttpTest): def test_ok(self): @@ -2246,6 +2349,7 @@ class TestResponseDict(MockHttpTest): ('delete_container', 'c'), ('post_object', 'c', 'o', {}), ('put_object', 'c', 'o', 'body'), + ('copy_object', 'c', 'o'), ('delete_object', 'c', 'o')] def fake_get_auth(*args, **kwargs):