openstack/python-swiftclient
Code Issues Proposed changes

Merge "Add option to skip container PUT during upload"

Browse Source
This commit is contained in:
Zuul 2022-08-23 00:37:52 +00:00 committed by Gerrit Code Review
commit 02509acc71
3 changed files with 102 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
swiftclient/service.py
View File

@ -202,6 +202,7 @@ _default_local_options = {
'leave_segments': False, 'leave_segments': False,
'changed': None, 'changed': None,
'skip_identical': False, 'skip_identical': False,
'skip_container_put': False,
'version_id': None, 'version_id': None,
'yes_all': False, 'yes_all': False,
'read_acl': None, 'read_acl': None,
@ -1465,6 +1466,7 @@ class SwiftService:
'leave_segments': False, 'leave_segments': False,
'changed': None, 'changed': None,
'skip_identical': False, 'skip_identical': False,
'skip_container_put': False,
'fail_fast': False, 'fail_fast': False,
'dir_marker': False # Only for None sources 'dir_marker': False # Only for None sources
} }
@ -1490,54 +1492,57 @@ class SwiftService:
# the object name. (same as passing --object-name). # the object name. (same as passing --object-name).
container, _sep, pseudo_folder = container.partition('/') container, _sep, pseudo_folder = container.partition('/')
# Try to create the container, just in case it doesn't exist. If this if not options['skip_container_put']:
# fails, it might just be because the user doesn't have container PUT # Try to create the container, just in case it doesn't exist. If
# permissions, so we'll ignore any error. If there's really a problem, # this fails, it might just be because the user doesn't have
# it'll surface on the first object PUT. # container PUT permissions, so we'll ignore any error. If there's
policy_header = {} # really a problem, it'll surface on the first object PUT.
_header = split_headers(options["header"]) policy_header = {}
if POLICY in _header: _header = split_headers(options["header"])
policy_header[POLICY] = \ if POLICY in _header:
_header[POLICY] policy_header[POLICY] = \
create_containers = [ _header[POLICY]
self.thread_manager.container_pool.submit( create_containers = [
self._create_container_job, container, headers=policy_header) self.thread_manager.container_pool.submit(
] self._create_container_job, container,
headers=policy_header)
]
# wait for first container job to complete before possibly attempting # wait for first container job to complete before possibly
# segment container job because segment container job may attempt # attempting segment container job because segment container job
# to HEAD the first container # may attempt to HEAD the first container
for r in interruptable_as_completed(create_containers): for r in interruptable_as_completed(create_containers):
res = r.result() res = r.result()
yield res yield res
if segment_size: if segment_size:
seg_container = container + '_segments' seg_container = container + '_segments'
if options['segment_container']: if options['segment_container']:
seg_container = options['segment_container'] seg_container = options['segment_container']
if seg_container != container: if seg_container != container:
if not policy_header: if not policy_header:
# Since no storage policy was specified on the command # Since no storage policy was specified on the command
# line, rather than just letting swift pick the default # line, rather than just letting swift pick the default
# storage policy, we'll try to create the segments # storage policy, we'll try to create the segments
# container with the same policy as the upload container # container with the same policy as the upload
create_containers = [ # container
self.thread_manager.container_pool.submit( create_containers = [
self._create_container_job, seg_container, self.thread_manager.container_pool.submit(
policy_source=container self._create_container_job, seg_container,
) policy_source=container
] )
else: ]
create_containers = [ else:
self.thread_manager.container_pool.submit( create_containers = [
self._create_container_job, seg_container, self.thread_manager.container_pool.submit(
headers=policy_header self._create_container_job, seg_container,
) headers=policy_header
] )
]
for r in interruptable_as_completed(create_containers): for r in interruptable_as_completed(create_containers):
res = r.result() res = r.result()
yield res yield res
# We maintain a results queue here and a separate thread to monitor # We maintain a results queue here and a separate thread to monitor
# the futures because we want to get results back from potential # the futures because we want to get results back from potential

13
swiftclient/shell.py
View File

@ -978,8 +978,9 @@ def st_copy(parser, args, output_manager, return_parser=False):
st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>] st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>]
[--segment-container <container>] [--leave-segments] [--segment-container <container>] [--leave-segments]
[--object-threads <thread>] [--segment-threads <threads>] [--object-threads <thread>] [--segment-threads <threads>]
[--meta <name:value>] [--header <header>] [--use-slo] [--meta <name:value>] [--header <header>]
[--ignore-checksum] [--object-name <object-name>] [--use-slo] [--ignore-checksum] [--skip-container-put]
[--object-name <object-name>]
<container> <file_or_directory> [<file_or_directory>] [...] <container> <file_or_directory> [<file_or_directory>] [...]
''' '''
@ -1025,11 +1026,13 @@ Optional arguments:
--use-slo When used in conjunction with --segment-size it will --use-slo When used in conjunction with --segment-size it will
create a Static Large Object instead of the default create a Static Large Object instead of the default
Dynamic Large Object. Dynamic Large Object.
--ignore-checksum Turn off checksum validation for uploads.
--skip-container-put Assume all necessary containers already exist; don't
automatically try to create them.
--object-name <object-name> --object-name <object-name>
Upload file and name object to <object-name> or upload Upload file and name object to <object-name> or upload
dir and use <object-name> as object prefix instead of dir and use <object-name> as object prefix instead of
folder name. folder name.
--ignore-checksum Turn off checksum validation for uploads.
'''.strip('\n') '''.strip('\n')
@ -1044,6 +1047,10 @@ def st_upload(parser, args, output_manager, return_parser=False):
'--skip-identical', action='store_true', dest='skip_identical', '--skip-identical', action='store_true', dest='skip_identical',
default=False, help='Skip uploading files that are identical on ' default=False, help='Skip uploading files that are identical on '
'both sides.') 'both sides.')
parser.add_argument(
'--skip-container-put', action='store_true', dest='skip_container_put',
default=False, help='Assume all necessary containers already exist; '
"don't automatically try to create them.")
parser.add_argument( parser.add_argument(
'-S', '--segment-size', dest='segment_size', help='Upload files ' '-S', '--segment-size', dest='segment_size', help='Upload files '
'in segments no larger than <size> (in Bytes) and then create a ' 'in segments no larger than <size> (in Bytes) and then create a '

42
test/unit/test_shell.py
View File

@ -907,6 +907,48 @@ class TestShell(unittest.TestCase):
query_string='multipart-manifest=put', query_string='multipart-manifest=put',
response_dict=mock.ANY) response_dict=mock.ANY)
@mock.patch('swiftclient.shell.walk')
@mock.patch('swiftclient.service.Connection')
def test_upload_skip_container_put(self, connection, walk):
connection.return_value.head_object.return_value = {
'content-length': '0'}
connection.return_value.put_object.return_value = EMPTY_ETAG
connection.return_value.attempts = 0
argv = ["", "upload", "container", "--skip-container-put",
self.tmpfile, "-H", "X-Storage-Policy:one",
"--meta", "Color:Blue"]
swiftclient.shell.main(argv)
connection.return_value.put_container.assert_not_called()
connection.return_value.put_object.assert_called_with(
'container',
self.tmpfile.lstrip('/'),
mock.ANY,
content_length=0,
headers={'x-object-meta-mtime': mock.ANY,
'X-Storage-Policy': 'one',
'X-Object-Meta-Color': 'Blue'},
response_dict={})
# Upload in segments
connection.return_value.head_container.return_value = {
'x-storage-policy': 'one'}
argv = ["", "upload", "container", "--skip-container-put",
self.tmpfile, "-S", "10"]
with open(self.tmpfile, "wb") as fh:
fh.write(b'12345678901234567890')
swiftclient.shell.main(argv)
# Both base and segments container are assumed to exist already
connection.return_value.put_container.assert_not_called()
connection.return_value.put_object.assert_called_with(
'container',
self.tmpfile.lstrip('/'),
'',
content_length=0,
headers={'x-object-manifest': mock.ANY,
'x-object-meta-mtime': mock.ANY},
response_dict={})
@mock.patch('swiftclient.service.SwiftService.upload') @mock.patch('swiftclient.service.SwiftService.upload')
def test_upload_object_with_account_readonly(self, upload): def test_upload_object_with_account_readonly(self, upload):
argv = ["", "upload", "container", self.tmpfile] argv = ["", "upload", "container", self.tmpfile]