From 5ed02345d36acf87fc4678e587db713004696124 Mon Sep 17 00:00:00 2001
From: James Nzomo <james@tdt.rocks>
Date: Sun, 24 Jan 2016 02:43:07 +0300
Subject: [PATCH] Fix segmented upload to pseudo-dir via <container>
This fix ensures creation and use of the correct default segment
container when pseudo-folder paths are passed via <container> arg.
Change-Id: I90356b041dc9dfbd55eb341271975621759476b9
Closes-Bug: 1532981
Related-Bug: 1478210
---
swiftclient/service.py | 25 +++++++++++++------------
tests/unit/test_service.py | 9 +++++++++
tests/unit/test_shell.py | 31 +++++++++++++++++++++++++++++--
3 files changed, 51 insertions(+), 14 deletions(-)
diff --git a/swiftclient/service.py b/swiftclient/service.py
index 09245d32..e092beca 100644
--- a/swiftclient/service.py
+++ b/swiftclient/service.py
@@ -25,6 +25,7 @@ from os import environ, makedirs, stat, utime
from os.path import (
basename, dirname, getmtime, getsize, isdir, join, sep as os_path_sep
)
+from posixpath import join as urljoin
from random import shuffle
from time import time
from threading import Thread
@@ -288,6 +289,7 @@ class SwiftUploadObject(object):
if not self.object_name:
raise SwiftError('Object names must not be empty strings')
+ self.object_name = self.object_name.lstrip('/')
self.options = options
self.source = source
@@ -1284,7 +1286,8 @@ class SwiftService(object):
"""
Upload a list of objects to a given container.
- :param container: The container to put the uploads into.
+ :param container: The container (or pseudo-folder path) to put the
+ uploads into.
:param objects: A list of file/directory names (strings) or
SwiftUploadObject instances containing a source for the
created object, an object name, and an options dict
@@ -1342,10 +1345,9 @@ class SwiftService(object):
raise SwiftError('Segment size should be an integer value')
# Incase we have a psudeo-folder path for <container> arg, derive
- # the container name from the top path to ensure new folder creation
- # and prevent spawning zero-byte objects shadowing pseudo-folders
- # by name.
- container_name = container.split('/', 1)[0]
+ # the container name from the top path and prepend the rest to
+ # the object name. (same as passing --object-name).
+ container, _sep, pseudo_folder = container.partition('/')
# Try to create the container, just in case it doesn't exist. If this
# fails, it might just be because the user doesn't have container PUT
@@ -1358,10 +1360,7 @@ class SwiftService(object):
_header[POLICY]
create_containers = [
self.thread_manager.container_pool.submit(
- self._create_container_job,
- container_name,
- headers=policy_header
- )
+ self._create_container_job, container, headers=policy_header)
]
# wait for first container job to complete before possibly attempting
@@ -1405,7 +1404,7 @@ class SwiftService(object):
rq = Queue()
file_jobs = {}
- upload_objects = self._make_upload_objects(objects)
+ upload_objects = self._make_upload_objects(objects, pseudo_folder)
for upload_object in upload_objects:
s = upload_object.source
o = upload_object.object_name
@@ -1496,14 +1495,16 @@ class SwiftService(object):
res = get_from_queue(rq)
@staticmethod
- def _make_upload_objects(objects):
+ def _make_upload_objects(objects, pseudo_folder=''):
upload_objects = []
for o in objects:
if isinstance(o, string_types):
- obj = SwiftUploadObject(o)
+ obj = SwiftUploadObject(o, urljoin(pseudo_folder,
+ o.lstrip('/')))
upload_objects.append(obj)
elif isinstance(o, SwiftUploadObject):
+ o.object_name = urljoin(pseudo_folder, o.object_name)
upload_objects.append(o)
else:
raise SwiftError(
diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py
index 003a51f8..c2a71434 100644
--- a/tests/unit/test_service.py
+++ b/tests/unit/test_service.py
@@ -1271,6 +1271,15 @@ class TestServiceUpload(_TestServiceBase):
]
mock_conn.get_container.assert_has_calls(expected)
+ def test_make_upload_objects(self):
+ # String list
+ filenames = ['/absolute/file/path', 'relative/file/path']
+ self.assertEqual(
+ [o.object_name for o in SwiftService._make_upload_objects(
+ filenames, 'pseudo/folder/path')],
+ ['pseudo/folder/path/absolute/file/path',
+ 'pseudo/folder/path/relative/file/path'])
+
class TestServiceDownload(_TestServiceBase):
diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py
index 01288c1f..1efc8dc0 100644
--- a/tests/unit/test_shell.py
+++ b/tests/unit/test_shell.py
@@ -485,8 +485,8 @@ class TestShell(testtools.TestCase):
response_dict={})
connection.return_value.put_object.assert_called_with(
- 'container/pseudo-folder/nested',
- self.tmpfile.lstrip('/'),
+ 'container',
+ 'pseudo-folder/nested' + self.tmpfile,
mock.ANY,
content_length=0,
headers={'x-object-meta-mtime': mock.ANY,
@@ -531,6 +531,33 @@ class TestShell(testtools.TestCase):
'x-object-meta-mtime': mock.ANY},
response_dict={})
+ # upload in segments to pseudo-folder (via <container> param)
+ connection.reset_mock()
+ connection.return_value.head_container.return_value = {
+ 'x-storage-policy': 'one'}
+ argv = ["", "upload", "container/pseudo-folder/nested",
+ self.tmpfile, "-S", "10", "--use-slo"]
+ with open(self.tmpfile, "wb") as fh:
+ fh.write(b'12345678901234567890')
+ swiftclient.shell.main(argv)
+ expected_calls = [mock.call('container',
+ {},
+ response_dict={}),
+ mock.call('container_segments',
+ {'X-Storage-Policy': 'one'},
+ response_dict={})]
+ connection.return_value.put_container.assert_has_calls(expected_calls)
+ connection.return_value.put_object.assert_called_with(
+ 'container',
+ 'pseudo-folder/nested' + self.tmpfile,
+ mock.ANY,
+ headers={
+ 'x-object-meta-mtime': mock.ANY,
+ 'x-static-large-object': 'true'
+ },
+ query_string='multipart-manifest=put',
+ response_dict={})
+
@mock.patch('swiftclient.service.SwiftService.upload')
def test_upload_object_with_account_readonly(self, upload):
argv = ["", "upload", "container", self.tmpfile]