Fix segmented upload to pseudo-dir via <container>

This fix ensures creation and use of the correct default segment
container when pseudo-folder paths are passed via <container> arg.

Change-Id: I90356b041dc9dfbd55eb341271975621759476b9
Closes-Bug: 1532981
Related-Bug: 1478210

swiftclient/service.py

@@ -25,6 +25,7 @@ from os import environ, makedirs, stat, utime
 from os.path import (
     basename, dirname, getmtime, getsize, isdir, join, sep as os_path_sep
 )
+from posixpath import join as urljoin
 from random import shuffle
 from time import time
 from threading import Thread
@@ -288,6 +289,7 @@ class SwiftUploadObject(object):
         if not self.object_name:
             raise SwiftError('Object names must not be empty strings')
 
+        self.object_name = self.object_name.lstrip('/')
         self.options = options
         self.source = source
 
@@ -1284,7 +1286,8 @@ class SwiftService(object):
         """
         Upload a list of objects to a given container.
 
-        :param container: The container to put the uploads into.
+        :param container: The container (or pseudo-folder path) to put the
+                          uploads into.
         :param objects: A list of file/directory names (strings) or
                         SwiftUploadObject instances containing a source for the
                         created object, an object name, and an options dict
@@ -1342,10 +1345,9 @@ class SwiftService(object):
             raise SwiftError('Segment size should be an integer value')
 
         # Incase we have a psudeo-folder path for <container> arg, derive
-        # the container name from the top path to ensure new folder creation
+        # the container name from the top path and prepend the rest to
-        # and prevent spawning zero-byte objects shadowing pseudo-folders
+        # the object name. (same as passing --object-name).
-        # by name.
+        container, _sep, pseudo_folder = container.partition('/')
-        container_name = container.split('/', 1)[0]
 
         # Try to create the container, just in case it doesn't exist. If this
         # fails, it might just be because the user doesn't have container PUT
@@ -1358,10 +1360,7 @@ class SwiftService(object):
                 _header[POLICY]
         create_containers = [
             self.thread_manager.container_pool.submit(
-                self._create_container_job,
+                self._create_container_job, container, headers=policy_header)
-                container_name,
-                headers=policy_header
-            )
         ]
 
         # wait for first container job to complete before possibly attempting
@@ -1405,7 +1404,7 @@ class SwiftService(object):
         rq = Queue()
         file_jobs = {}
 
-        upload_objects = self._make_upload_objects(objects)
+        upload_objects = self._make_upload_objects(objects, pseudo_folder)
         for upload_object in upload_objects:
             s = upload_object.source
             o = upload_object.object_name
@@ -1496,14 +1495,16 @@ class SwiftService(object):
             res = get_from_queue(rq)
 
     @staticmethod
-    def _make_upload_objects(objects):
+    def _make_upload_objects(objects, pseudo_folder=''):
         upload_objects = []
 
         for o in objects:
             if isinstance(o, string_types):
-                obj = SwiftUploadObject(o)
+                obj = SwiftUploadObject(o, urljoin(pseudo_folder,
+                                                   o.lstrip('/')))
                 upload_objects.append(obj)
             elif isinstance(o, SwiftUploadObject):
+                o.object_name = urljoin(pseudo_folder, o.object_name)
                 upload_objects.append(o)
             else:
                 raise SwiftError(

tests/unit/test_service.py

@@ -1271,6 +1271,15 @@ class TestServiceUpload(_TestServiceBase):
             ]
             mock_conn.get_container.assert_has_calls(expected)
 
+    def test_make_upload_objects(self):
+        # String list
+        filenames = ['/absolute/file/path', 'relative/file/path']
+        self.assertEqual(
+            [o.object_name for o in SwiftService._make_upload_objects(
+             filenames, 'pseudo/folder/path')],
+            ['pseudo/folder/path/absolute/file/path',
+             'pseudo/folder/path/relative/file/path'])
+
 
 class TestServiceDownload(_TestServiceBase):
 

tests/unit/test_shell.py

@@ -485,8 +485,8 @@ class TestShell(testtools.TestCase):
             response_dict={})
 
         connection.return_value.put_object.assert_called_with(
-            'container/pseudo-folder/nested',
+            'container',
-            self.tmpfile.lstrip('/'),
+            'pseudo-folder/nested' + self.tmpfile,
             mock.ANY,
             content_length=0,
             headers={'x-object-meta-mtime': mock.ANY,
@@ -531,6 +531,33 @@ class TestShell(testtools.TestCase):
                      'x-object-meta-mtime': mock.ANY},
             response_dict={})
 
+        # upload in segments to pseudo-folder (via <container> param)
+        connection.reset_mock()
+        connection.return_value.head_container.return_value = {
+            'x-storage-policy': 'one'}
+        argv = ["", "upload", "container/pseudo-folder/nested",
+                self.tmpfile, "-S", "10", "--use-slo"]
+        with open(self.tmpfile, "wb") as fh:
+            fh.write(b'12345678901234567890')
+        swiftclient.shell.main(argv)
+        expected_calls = [mock.call('container',
+                                    {},
+                                    response_dict={}),
+                          mock.call('container_segments',
+                                    {'X-Storage-Policy': 'one'},
+                                    response_dict={})]
+        connection.return_value.put_container.assert_has_calls(expected_calls)
+        connection.return_value.put_object.assert_called_with(
+            'container',
+            'pseudo-folder/nested' + self.tmpfile,
+            mock.ANY,
+            headers={
+                'x-object-meta-mtime': mock.ANY,
+                'x-static-large-object': 'true'
+            },
+            query_string='multipart-manifest=put',
+            response_dict={})
+
     @mock.patch('swiftclient.service.SwiftService.upload')
     def test_upload_object_with_account_readonly(self, upload):
         argv = ["", "upload", "container", self.tmpfile]