Merge "Add an option: disable etag check on downloads"

doc/manpages/swift.1

@@ -86,6 +86,7 @@ container, or a list of objects depending on the args given. For a single
 object download, you may use the \-o [\-\-output] <filename> option to
 redirect the output to a specific file or if "-" then just redirect to stdout or
 with \-\-no-download actually not to write anything to disk.
+The \-\-ignore-checksum is an option that turn off checksum validation.
 You can specify optional headers with the repeatable cURL-like option
 \-H [\-\-header]. For more details and options see swift download \-\-help.
 .RE

doc/source/cli.rst

@@ -186,7 +186,8 @@ Download
        container, or a list of objects depending on the arguments given. For a
        single object download, you may use the ``-o <filename>`` or ``--output <filename>``
        option to redirect the output to a specific file or ``-`` to
-       redirect to stdout. You can specify optional headers with the repeatable
+       redirect to stdout. The ``--ignore-checksum`` is an option that turn off
+       checksum validation. You can specify optional headers with the repeatable
        cURL-like option ``-H [--header <name:value>]``.
 
 Delete

doc/source/service-api.rst

@@ -207,6 +207,9 @@ Options
         Affects uploads, and allows empty 'pseudofolder' objects to be created
         when the source of an upload is ``None``.
 
+    ``checksum``: ``True``
+        Affects uploads and downloads, and means if check md5 for downloads/uploads.
+
     ``shuffle``: ``False``
         When downloading objects, the default behaviour of the CLI is to shuffle
         lists of objects in order to spread the load on storage drives when multiple

swiftclient/service.py

@@ -336,7 +336,7 @@ class _SwiftReader(object):
     errors on failures caused by either invalid md5sum or size of the
     data read.
     """
-    def __init__(self, path, body, headers):
+    def __init__(self, path, body, headers, checksum=True):
         self._path = path
         self._body = body
         self._actual_read = 0
@@ -345,7 +345,7 @@ class _SwiftReader(object):
         self._expected_etag = headers.get('etag')
 
         if ('x-object-manifest' not in headers
-                and 'x-static-large-object' not in headers):
+                and 'x-static-large-object' not in headers and checksum):
             self._actual_md5 = md5()
 
         if 'content-length' in headers:
@@ -980,6 +980,7 @@ class SwiftService(object):
                                 'header': [],
                                 'skip_identical': False,
                                 'out_directory': None,
+                                'checksum': True,
                                 'out_file': None,
                                 'remove_prefix': False,
                                 'shuffle' : False
@@ -1135,7 +1136,8 @@ class SwiftService(object):
 
             headers_receipt = time()
 
-            obj_body = _SwiftReader(path, body, headers)
+            obj_body = _SwiftReader(path, body, headers,
+                                    options.get('checksum', True))
 
             no_file = options['no_download']
             if out_file == "-" and not no_file:

swiftclient/shell.py

@@ -204,7 +204,7 @@ def st_delete(parser, args, output_manager):
 
 st_download_options = '''[--all] [--marker <marker>] [--prefix <prefix>]
                       [--output <out_file>] [--output-dir <out_directory>]
-                      [--object-threads <threads>]
+                      [--object-threads <threads>] [--ignore-checksum]
                       [--container-threads <threads>] [--no-download]
                       [--skip-identical] [--remove-prefix]
                       [--header <header:value>] [--no-shuffle]
@@ -251,6 +251,7 @@ Optional arguments:
                         Example --header "content-type:text/plain"
   --skip-identical      Skip downloading files that are identical on both
                         sides.
+  --ignore-checksum     Turn off checksum validation for downloads.
   --no-shuffle          By default, when downloading a complete account or
                         container, download order is randomised in order to
                         reduce the load on individual drives when multiple
@@ -308,6 +309,9 @@ def st_download(parser, args, output_manager):
         '--skip-identical', action='store_true', dest='skip_identical',
         default=False, help='Skip downloading files that are identical on '
         'both sides.')
+    parser.add_argument(
+        '--ignore-checksum', action='store_false', dest='checksum',
+        default=True, help='Turn off checksum validation for downloads.')
     parser.add_argument(
         '--no-shuffle', action='store_false', dest='shuffle',
         default=True, help='By default, download order is randomised in order '

tests/unit/test_service.py

@@ -103,6 +103,15 @@ class TestSwiftReader(unittest.TestCase):
         self.assertEqual(sr._expected_etag, None)
         self.assertEqual(sr._actual_md5, None)
 
+    def test_create_with_ignore_checksum(self):
+        # md5 should not be initialized if checksum is False
+        sr = self.sr('path', 'body', {}, False)
+        self.assertEqual(sr._path, 'path')
+        self.assertEqual(sr._body, 'body')
+        self.assertEqual(sr._content_length, None)
+        self.assertEqual(sr._expected_etag, None)
+        self.assertEqual(sr._actual_md5, None)
+
     def test_create_with_content_length(self):
         sr = self.sr('path', 'body', {'content-length': 5})
 

tests/unit/test_shell.py

@@ -367,6 +367,24 @@ class TestShell(unittest.TestCase):
         mock_open.assert_called_with('object', 'wb')
         self.assertEqual([], makedirs.mock_calls)
 
+        # Test downloading without md5 checks
+        objcontent = six.BytesIO(b'objcontent')
+        connection.return_value.get_object.side_effect = [
+            ({'content-type': 'text/plain',
+              'etag': '2cbbfe139a744d6abbe695e17f3c1991'},
+             objcontent)
+        ]
+        with mock.patch(BUILTIN_OPEN) as mock_open, mock.patch(
+                'swiftclient.service._SwiftReader') as sr:
+            argv = ["", "download", "container", "object", "--ignore-check"]
+            swiftclient.shell.main(argv)
+        connection.return_value.get_object.assert_called_with(
+            'container', 'object', headers={}, resp_chunk_size=65536,
+            response_dict={})
+        mock_open.assert_called_with('object', 'wb')
+        sr.assert_called_once_with('object', mock.ANY, mock.ANY, False)
+        self.assertEqual([], makedirs.mock_calls)
+
         # Test downloading single object to stdout
         objcontent = six.BytesIO(b'objcontent')
         connection.return_value.get_object.side_effect = [