Merge "Add option to skip container PUT during upload"
This commit is contained in:
commit
02509acc71
@ -202,6 +202,7 @@ _default_local_options = {
|
|||||||
'leave_segments': False,
|
'leave_segments': False,
|
||||||
'changed': None,
|
'changed': None,
|
||||||
'skip_identical': False,
|
'skip_identical': False,
|
||||||
|
'skip_container_put': False,
|
||||||
'version_id': None,
|
'version_id': None,
|
||||||
'yes_all': False,
|
'yes_all': False,
|
||||||
'read_acl': None,
|
'read_acl': None,
|
||||||
@ -1465,6 +1466,7 @@ class SwiftService:
|
|||||||
'leave_segments': False,
|
'leave_segments': False,
|
||||||
'changed': None,
|
'changed': None,
|
||||||
'skip_identical': False,
|
'skip_identical': False,
|
||||||
|
'skip_container_put': False,
|
||||||
'fail_fast': False,
|
'fail_fast': False,
|
||||||
'dir_marker': False # Only for None sources
|
'dir_marker': False # Only for None sources
|
||||||
}
|
}
|
||||||
@ -1490,54 +1492,57 @@ class SwiftService:
|
|||||||
# the object name. (same as passing --object-name).
|
# the object name. (same as passing --object-name).
|
||||||
container, _sep, pseudo_folder = container.partition('/')
|
container, _sep, pseudo_folder = container.partition('/')
|
||||||
|
|
||||||
# Try to create the container, just in case it doesn't exist. If this
|
if not options['skip_container_put']:
|
||||||
# fails, it might just be because the user doesn't have container PUT
|
# Try to create the container, just in case it doesn't exist. If
|
||||||
# permissions, so we'll ignore any error. If there's really a problem,
|
# this fails, it might just be because the user doesn't have
|
||||||
# it'll surface on the first object PUT.
|
# container PUT permissions, so we'll ignore any error. If there's
|
||||||
policy_header = {}
|
# really a problem, it'll surface on the first object PUT.
|
||||||
_header = split_headers(options["header"])
|
policy_header = {}
|
||||||
if POLICY in _header:
|
_header = split_headers(options["header"])
|
||||||
policy_header[POLICY] = \
|
if POLICY in _header:
|
||||||
_header[POLICY]
|
policy_header[POLICY] = \
|
||||||
create_containers = [
|
_header[POLICY]
|
||||||
self.thread_manager.container_pool.submit(
|
create_containers = [
|
||||||
self._create_container_job, container, headers=policy_header)
|
self.thread_manager.container_pool.submit(
|
||||||
]
|
self._create_container_job, container,
|
||||||
|
headers=policy_header)
|
||||||
|
]
|
||||||
|
|
||||||
# wait for first container job to complete before possibly attempting
|
# wait for first container job to complete before possibly
|
||||||
# segment container job because segment container job may attempt
|
# attempting segment container job because segment container job
|
||||||
# to HEAD the first container
|
# may attempt to HEAD the first container
|
||||||
for r in interruptable_as_completed(create_containers):
|
for r in interruptable_as_completed(create_containers):
|
||||||
res = r.result()
|
res = r.result()
|
||||||
yield res
|
yield res
|
||||||
|
|
||||||
if segment_size:
|
if segment_size:
|
||||||
seg_container = container + '_segments'
|
seg_container = container + '_segments'
|
||||||
if options['segment_container']:
|
if options['segment_container']:
|
||||||
seg_container = options['segment_container']
|
seg_container = options['segment_container']
|
||||||
if seg_container != container:
|
if seg_container != container:
|
||||||
if not policy_header:
|
if not policy_header:
|
||||||
# Since no storage policy was specified on the command
|
# Since no storage policy was specified on the command
|
||||||
# line, rather than just letting swift pick the default
|
# line, rather than just letting swift pick the default
|
||||||
# storage policy, we'll try to create the segments
|
# storage policy, we'll try to create the segments
|
||||||
# container with the same policy as the upload container
|
# container with the same policy as the upload
|
||||||
create_containers = [
|
# container
|
||||||
self.thread_manager.container_pool.submit(
|
create_containers = [
|
||||||
self._create_container_job, seg_container,
|
self.thread_manager.container_pool.submit(
|
||||||
policy_source=container
|
self._create_container_job, seg_container,
|
||||||
)
|
policy_source=container
|
||||||
]
|
)
|
||||||
else:
|
]
|
||||||
create_containers = [
|
else:
|
||||||
self.thread_manager.container_pool.submit(
|
create_containers = [
|
||||||
self._create_container_job, seg_container,
|
self.thread_manager.container_pool.submit(
|
||||||
headers=policy_header
|
self._create_container_job, seg_container,
|
||||||
)
|
headers=policy_header
|
||||||
]
|
)
|
||||||
|
]
|
||||||
|
|
||||||
for r in interruptable_as_completed(create_containers):
|
for r in interruptable_as_completed(create_containers):
|
||||||
res = r.result()
|
res = r.result()
|
||||||
yield res
|
yield res
|
||||||
|
|
||||||
# We maintain a results queue here and a separate thread to monitor
|
# We maintain a results queue here and a separate thread to monitor
|
||||||
# the futures because we want to get results back from potential
|
# the futures because we want to get results back from potential
|
||||||
|
@ -978,8 +978,9 @@ def st_copy(parser, args, output_manager, return_parser=False):
|
|||||||
st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>]
|
st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>]
|
||||||
[--segment-container <container>] [--leave-segments]
|
[--segment-container <container>] [--leave-segments]
|
||||||
[--object-threads <thread>] [--segment-threads <threads>]
|
[--object-threads <thread>] [--segment-threads <threads>]
|
||||||
[--meta <name:value>] [--header <header>] [--use-slo]
|
[--meta <name:value>] [--header <header>]
|
||||||
[--ignore-checksum] [--object-name <object-name>]
|
[--use-slo] [--ignore-checksum] [--skip-container-put]
|
||||||
|
[--object-name <object-name>]
|
||||||
<container> <file_or_directory> [<file_or_directory>] [...]
|
<container> <file_or_directory> [<file_or_directory>] [...]
|
||||||
'''
|
'''
|
||||||
|
|
||||||
@ -1025,11 +1026,13 @@ Optional arguments:
|
|||||||
--use-slo When used in conjunction with --segment-size it will
|
--use-slo When used in conjunction with --segment-size it will
|
||||||
create a Static Large Object instead of the default
|
create a Static Large Object instead of the default
|
||||||
Dynamic Large Object.
|
Dynamic Large Object.
|
||||||
|
--ignore-checksum Turn off checksum validation for uploads.
|
||||||
|
--skip-container-put Assume all necessary containers already exist; don't
|
||||||
|
automatically try to create them.
|
||||||
--object-name <object-name>
|
--object-name <object-name>
|
||||||
Upload file and name object to <object-name> or upload
|
Upload file and name object to <object-name> or upload
|
||||||
dir and use <object-name> as object prefix instead of
|
dir and use <object-name> as object prefix instead of
|
||||||
folder name.
|
folder name.
|
||||||
--ignore-checksum Turn off checksum validation for uploads.
|
|
||||||
'''.strip('\n')
|
'''.strip('\n')
|
||||||
|
|
||||||
|
|
||||||
@ -1044,6 +1047,10 @@ def st_upload(parser, args, output_manager, return_parser=False):
|
|||||||
'--skip-identical', action='store_true', dest='skip_identical',
|
'--skip-identical', action='store_true', dest='skip_identical',
|
||||||
default=False, help='Skip uploading files that are identical on '
|
default=False, help='Skip uploading files that are identical on '
|
||||||
'both sides.')
|
'both sides.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--skip-container-put', action='store_true', dest='skip_container_put',
|
||||||
|
default=False, help='Assume all necessary containers already exist; '
|
||||||
|
"don't automatically try to create them.")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'-S', '--segment-size', dest='segment_size', help='Upload files '
|
'-S', '--segment-size', dest='segment_size', help='Upload files '
|
||||||
'in segments no larger than <size> (in Bytes) and then create a '
|
'in segments no larger than <size> (in Bytes) and then create a '
|
||||||
|
@ -907,6 +907,48 @@ class TestShell(unittest.TestCase):
|
|||||||
query_string='multipart-manifest=put',
|
query_string='multipart-manifest=put',
|
||||||
response_dict=mock.ANY)
|
response_dict=mock.ANY)
|
||||||
|
|
||||||
|
@mock.patch('swiftclient.shell.walk')
|
||||||
|
@mock.patch('swiftclient.service.Connection')
|
||||||
|
def test_upload_skip_container_put(self, connection, walk):
|
||||||
|
connection.return_value.head_object.return_value = {
|
||||||
|
'content-length': '0'}
|
||||||
|
connection.return_value.put_object.return_value = EMPTY_ETAG
|
||||||
|
connection.return_value.attempts = 0
|
||||||
|
argv = ["", "upload", "container", "--skip-container-put",
|
||||||
|
self.tmpfile, "-H", "X-Storage-Policy:one",
|
||||||
|
"--meta", "Color:Blue"]
|
||||||
|
swiftclient.shell.main(argv)
|
||||||
|
connection.return_value.put_container.assert_not_called()
|
||||||
|
|
||||||
|
connection.return_value.put_object.assert_called_with(
|
||||||
|
'container',
|
||||||
|
self.tmpfile.lstrip('/'),
|
||||||
|
mock.ANY,
|
||||||
|
content_length=0,
|
||||||
|
headers={'x-object-meta-mtime': mock.ANY,
|
||||||
|
'X-Storage-Policy': 'one',
|
||||||
|
'X-Object-Meta-Color': 'Blue'},
|
||||||
|
response_dict={})
|
||||||
|
|
||||||
|
# Upload in segments
|
||||||
|
connection.return_value.head_container.return_value = {
|
||||||
|
'x-storage-policy': 'one'}
|
||||||
|
argv = ["", "upload", "container", "--skip-container-put",
|
||||||
|
self.tmpfile, "-S", "10"]
|
||||||
|
with open(self.tmpfile, "wb") as fh:
|
||||||
|
fh.write(b'12345678901234567890')
|
||||||
|
swiftclient.shell.main(argv)
|
||||||
|
# Both base and segments container are assumed to exist already
|
||||||
|
connection.return_value.put_container.assert_not_called()
|
||||||
|
connection.return_value.put_object.assert_called_with(
|
||||||
|
'container',
|
||||||
|
self.tmpfile.lstrip('/'),
|
||||||
|
'',
|
||||||
|
content_length=0,
|
||||||
|
headers={'x-object-manifest': mock.ANY,
|
||||||
|
'x-object-meta-mtime': mock.ANY},
|
||||||
|
response_dict={})
|
||||||
|
|
||||||
@mock.patch('swiftclient.service.SwiftService.upload')
|
@mock.patch('swiftclient.service.SwiftService.upload')
|
||||||
def test_upload_object_with_account_readonly(self, upload):
|
def test_upload_object_with_account_readonly(self, upload):
|
||||||
argv = ["", "upload", "container", self.tmpfile]
|
argv = ["", "upload", "container", self.tmpfile]
|
||||||
|
Loading…
x
Reference in New Issue
Block a user