Merge "Add option to skip container PUT during upload"

swiftclient/service.py

@@ -202,6 +202,7 @@ _default_local_options = {
     'leave_segments': False,
     'changed': None,
     'skip_identical': False,
+    'skip_container_put': False,
     'version_id': None,
     'yes_all': False,
     'read_acl': None,
@@ -1465,6 +1466,7 @@ class SwiftService:
                                 'leave_segments': False,
                                 'changed': None,
                                 'skip_identical': False,
+                                'skip_container_put': False,
                                 'fail_fast': False,
                                 'dir_marker': False  # Only for None sources
                             }
@@ -1490,54 +1492,57 @@ class SwiftService:
         # the object name. (same as passing --object-name).
         container, _sep, pseudo_folder = container.partition('/')
 
-        # Try to create the container, just in case it doesn't exist. If this
+        if not options['skip_container_put']:
-        # fails, it might just be because the user doesn't have container PUT
+            # Try to create the container, just in case it doesn't exist. If
-        # permissions, so we'll ignore any error. If there's really a problem,
+            # this fails, it might just be because the user doesn't have
-        # it'll surface on the first object PUT.
+            # container PUT permissions, so we'll ignore any error. If there's
-        policy_header = {}
+            # really a problem, it'll surface on the first object PUT.
-        _header = split_headers(options["header"])
+            policy_header = {}
-        if POLICY in _header:
+            _header = split_headers(options["header"])
-            policy_header[POLICY] = \
+            if POLICY in _header:
-                _header[POLICY]
+                policy_header[POLICY] = \
-        create_containers = [
+                    _header[POLICY]
-            self.thread_manager.container_pool.submit(
+            create_containers = [
-                self._create_container_job, container, headers=policy_header)
+                self.thread_manager.container_pool.submit(
-        ]
+                    self._create_container_job, container,
+                    headers=policy_header)
+            ]
 
-        # wait for first container job to complete before possibly attempting
+            # wait for first container job to complete before possibly
-        # segment container job because segment container job may attempt
+            # attempting segment container job because segment container job
-        # to HEAD the first container
+            # may attempt to HEAD the first container
-        for r in interruptable_as_completed(create_containers):
+            for r in interruptable_as_completed(create_containers):
-            res = r.result()
+                res = r.result()
-            yield res
+                yield res
 
-        if segment_size:
+            if segment_size:
-            seg_container = container + '_segments'
+                seg_container = container + '_segments'
-            if options['segment_container']:
+                if options['segment_container']:
-                seg_container = options['segment_container']
+                    seg_container = options['segment_container']
-            if seg_container != container:
+                if seg_container != container:
-                if not policy_header:
+                    if not policy_header:
-                    # Since no storage policy was specified on the command
+                        # Since no storage policy was specified on the command
-                    # line, rather than just letting swift pick the default
+                        # line, rather than just letting swift pick the default
-                    # storage policy, we'll try to create the segments
+                        # storage policy, we'll try to create the segments
-                    # container with the same policy as the upload container
+                        # container with the same policy as the upload
-                    create_containers = [
+                        # container
-                        self.thread_manager.container_pool.submit(
+                        create_containers = [
-                            self._create_container_job, seg_container,
+                            self.thread_manager.container_pool.submit(
-                            policy_source=container
+                                self._create_container_job, seg_container,
-                        )
+                                policy_source=container
-                    ]
+                            )
-                else:
+                        ]
-                    create_containers = [
+                    else:
-                        self.thread_manager.container_pool.submit(
+                        create_containers = [
-                            self._create_container_job, seg_container,
+                            self.thread_manager.container_pool.submit(
-                            headers=policy_header
+                                self._create_container_job, seg_container,
-                        )
+                                headers=policy_header
-                    ]
+                            )
+                        ]
 
-                for r in interruptable_as_completed(create_containers):
+                    for r in interruptable_as_completed(create_containers):
-                    res = r.result()
+                        res = r.result()
-                    yield res
+                        yield res
 
         # We maintain a results queue here and a separate thread to monitor
         # the futures because we want to get results back from potential

swiftclient/shell.py

@@ -978,8 +978,9 @@ def st_copy(parser, args, output_manager, return_parser=False):
 st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>]
                     [--segment-container <container>] [--leave-segments]
                     [--object-threads <thread>] [--segment-threads <threads>]
-                    [--meta <name:value>] [--header <header>] [--use-slo]
+                    [--meta <name:value>] [--header <header>]
-                    [--ignore-checksum] [--object-name <object-name>]
+                    [--use-slo] [--ignore-checksum] [--skip-container-put]
+                    [--object-name <object-name>]
                     <container> <file_or_directory> [<file_or_directory>] [...]
 '''
 
@@ -1025,11 +1026,13 @@ Optional arguments:
   --use-slo             When used in conjunction with --segment-size it will
                         create a Static Large Object instead of the default
                         Dynamic Large Object.
+  --ignore-checksum     Turn off checksum validation for uploads.
+  --skip-container-put  Assume all necessary containers already exist; don't
+                        automatically try to create them.
   --object-name <object-name>
                         Upload file and name object to <object-name> or upload
                         dir and use <object-name> as object prefix instead of
                         folder name.
-  --ignore-checksum     Turn off checksum validation for uploads.
 '''.strip('\n')
 
 
@@ -1044,6 +1047,10 @@ def st_upload(parser, args, output_manager, return_parser=False):
         '--skip-identical', action='store_true', dest='skip_identical',
         default=False, help='Skip uploading files that are identical on '
         'both sides.')
+    parser.add_argument(
+        '--skip-container-put', action='store_true', dest='skip_container_put',
+        default=False, help='Assume all necessary containers already exist; '
+        "don't automatically try to create them.")
     parser.add_argument(
         '-S', '--segment-size', dest='segment_size', help='Upload files '
         'in segments no larger than <size> (in Bytes) and then create a '

test/unit/test_shell.py

@@ -907,6 +907,48 @@ class TestShell(unittest.TestCase):
             query_string='multipart-manifest=put',
             response_dict=mock.ANY)
 
+    @mock.patch('swiftclient.shell.walk')
+    @mock.patch('swiftclient.service.Connection')
+    def test_upload_skip_container_put(self, connection, walk):
+        connection.return_value.head_object.return_value = {
+            'content-length': '0'}
+        connection.return_value.put_object.return_value = EMPTY_ETAG
+        connection.return_value.attempts = 0
+        argv = ["", "upload", "container", "--skip-container-put",
+                self.tmpfile, "-H", "X-Storage-Policy:one",
+                "--meta", "Color:Blue"]
+        swiftclient.shell.main(argv)
+        connection.return_value.put_container.assert_not_called()
+
+        connection.return_value.put_object.assert_called_with(
+            'container',
+            self.tmpfile.lstrip('/'),
+            mock.ANY,
+            content_length=0,
+            headers={'x-object-meta-mtime': mock.ANY,
+                     'X-Storage-Policy': 'one',
+                     'X-Object-Meta-Color': 'Blue'},
+            response_dict={})
+
+        # Upload in segments
+        connection.return_value.head_container.return_value = {
+            'x-storage-policy': 'one'}
+        argv = ["", "upload", "container", "--skip-container-put",
+                self.tmpfile, "-S", "10"]
+        with open(self.tmpfile, "wb") as fh:
+            fh.write(b'12345678901234567890')
+        swiftclient.shell.main(argv)
+        # Both base and segments container are assumed to exist already
+        connection.return_value.put_container.assert_not_called()
+        connection.return_value.put_object.assert_called_with(
+            'container',
+            self.tmpfile.lstrip('/'),
+            '',
+            content_length=0,
+            headers={'x-object-manifest': mock.ANY,
+                     'x-object-meta-mtime': mock.ANY},
+            response_dict={})
+
     @mock.patch('swiftclient.service.SwiftService.upload')
     def test_upload_object_with_account_readonly(self, upload):
         argv = ["", "upload", "container", self.tmpfile]