Allow for uploads from standard input.
If "-" is passed in for the source, python-swiftclient will upload the object by reading the contents of the standard input. The object name option must be set, as well, and this cannot be used in conjunction with other files. This approach stores the entire contents as one object. A follow on patch will change this behavior to upload from standard input as SLO, unless the segment size is larger than the content size. Change-Id: I1a8be6377de06f702e0f336a5a593408ed49be02
This commit is contained in:
parent
124c7de676
commit
0982791db2
@ -63,8 +63,11 @@ Uploads to the given container the files and directories specified by the
|
||||
remaining args. The \-c or \-\-changed is an option that will only upload files
|
||||
that have changed since the last upload. The \-\-object\-name <object\-name> is
|
||||
an option that will upload file and name object to <object\-name> or upload dir
|
||||
and use <object\-name> as object prefix. The \-S <size> or \-\-segment\-size <size>
|
||||
and \-\-leave\-segments and others are options as well (see swift upload \-\-help for more).
|
||||
and use <object\-name> as object prefix. If the file name is "-", reads the
|
||||
content from standard input. In this case, \-\-object\-name is required and no
|
||||
other files may be given. The \-S <size> or \-\-segment\-size <size> and
|
||||
\-\-leave\-segments and others are options as well (see swift upload \-\-help
|
||||
for more).
|
||||
.RE
|
||||
|
||||
\fBpost\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR]
|
||||
|
@ -369,10 +369,10 @@ given container. The ``-c`` or ``--changed`` is an option that will only
|
||||
upload files that have changed since the last upload. The
|
||||
``--object-name <object-name>`` is an option that will upload a file and
|
||||
name object to ``<object-name>`` or upload a directory and use ``<object-name>``
|
||||
as object prefix. The ``-S <size>`` or ``--segment-size <size>`` and
|
||||
``--leave-segments`` are options as well (see ``--help`` for more).
|
||||
|
||||
Uploads specified files and directories to the given container.
|
||||
as object prefix. If the file name is "-", client reads content from standard
|
||||
input. In this case ``--object-name`` is required to set the name of the object
|
||||
and no other files may be given. The ``-S <size>`` or ``--segment-size <size>``
|
||||
and ``--leave-segments`` are options as well (see ``--help`` for more).
|
||||
|
||||
**Positional arguments:**
|
||||
|
||||
|
@ -1811,6 +1811,8 @@ class SwiftService(object):
|
||||
return chunks
|
||||
|
||||
def _is_identical(self, chunk_data, path):
|
||||
if path is None:
|
||||
return False
|
||||
try:
|
||||
fp = open(path, 'rb', DISK_BUFFER)
|
||||
except IOError:
|
||||
|
@ -17,6 +17,7 @@
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import signal
|
||||
@ -26,7 +27,7 @@ from os import environ, walk, _exit as os_exit
|
||||
from os.path import isfile, isdir, join
|
||||
from six import text_type, PY2
|
||||
from six.moves.urllib.parse import unquote, urlparse
|
||||
from sys import argv as sys_argv, exit, stderr
|
||||
from sys import argv as sys_argv, exit, stderr, stdin
|
||||
from time import gmtime, strftime
|
||||
|
||||
from swiftclient import RequestException
|
||||
@ -901,7 +902,9 @@ Uploads specified files and directories to the given container.
|
||||
Positional arguments:
|
||||
<container> Name of container to upload to.
|
||||
<file_or_directory> Name of file or directory to upload. Specify multiple
|
||||
times for multiple uploads.
|
||||
times for multiple uploads. If "-" is specified, reads
|
||||
content from standard input (--object-name is required
|
||||
in this case).
|
||||
|
||||
Optional arguments:
|
||||
-c, --changed Only upload files that have changed since the last
|
||||
@ -1002,6 +1005,11 @@ def st_upload(parser, args, output_manager):
|
||||
else:
|
||||
container = args[0]
|
||||
files = args[1:]
|
||||
from_stdin = '-' in files
|
||||
if from_stdin and len(files) > 1:
|
||||
output_manager.error(
|
||||
'upload from stdin cannot be used along with other files')
|
||||
return
|
||||
|
||||
if options['object_name'] is not None:
|
||||
if len(files) > 1:
|
||||
@ -1009,6 +1017,10 @@ def st_upload(parser, args, output_manager):
|
||||
return
|
||||
else:
|
||||
orig_path = files[0]
|
||||
elif from_stdin:
|
||||
output_manager.error(
|
||||
'object-name must be specified with uploads from stdin')
|
||||
return
|
||||
|
||||
if options['segment_size']:
|
||||
try:
|
||||
@ -1047,6 +1059,14 @@ def st_upload(parser, args, output_manager):
|
||||
objs = []
|
||||
dir_markers = []
|
||||
for f in files:
|
||||
if f == '-':
|
||||
fd = io.open(stdin.fileno(), mode='rb')
|
||||
objs.append(SwiftUploadObject(
|
||||
fd, object_name=options['object_name']))
|
||||
# We ensure that there is exactly one "file" to upload in
|
||||
# this case -- stdin
|
||||
break
|
||||
|
||||
if isfile(f):
|
||||
objs.append(f)
|
||||
elif isdir(f):
|
||||
@ -1060,7 +1080,7 @@ def st_upload(parser, args, output_manager):
|
||||
|
||||
# Now that we've collected all the required files and dir markers
|
||||
# build the tuples for the call to upload
|
||||
if options['object_name'] is not None:
|
||||
if options['object_name'] is not None and not from_stdin:
|
||||
objs = [
|
||||
SwiftUploadObject(
|
||||
o, object_name=o.replace(
|
||||
|
@ -27,6 +27,7 @@ from time import localtime, mktime, strftime, strptime
|
||||
|
||||
from requests.packages.urllib3.exceptions import InsecureRequestWarning
|
||||
import six
|
||||
import sys
|
||||
|
||||
import swiftclient
|
||||
from swiftclient.service import SwiftError
|
||||
@ -719,7 +720,7 @@ class TestShell(unittest.TestCase):
|
||||
'x-object-meta-mtime': mock.ANY,
|
||||
},
|
||||
query_string='multipart-manifest=put',
|
||||
response_dict={})
|
||||
response_dict=mock.ANY)
|
||||
|
||||
@mock.patch('swiftclient.service.SwiftService.upload')
|
||||
def test_upload_object_with_account_readonly(self, upload):
|
||||
@ -905,6 +906,44 @@ class TestShell(unittest.TestCase):
|
||||
'x-object-meta-mtime': mock.ANY},
|
||||
response_dict={})
|
||||
|
||||
@mock.patch('swiftclient.shell.io.open')
|
||||
@mock.patch('swiftclient.service.SwiftService.upload')
|
||||
def test_upload_from_stdin(self, upload_mock, io_open_mock):
|
||||
def fake_open(fd, mode):
|
||||
mock_io = mock.Mock()
|
||||
mock_io.fileno.return_value = fd
|
||||
return mock_io
|
||||
|
||||
io_open_mock.side_effect = fake_open
|
||||
|
||||
argv = ["", "upload", "container", "-", "--object-name", "foo"]
|
||||
swiftclient.shell.main(argv)
|
||||
upload_mock.assert_called_once_with("container", mock.ANY)
|
||||
# This is a little convoluted: we want to examine the first call ([0]),
|
||||
# the argv list([1]), the second parameter ([1]), and the first
|
||||
# element. This is because the upload method takes a container and a
|
||||
# list of SwiftUploadObjects.
|
||||
swift_upload_obj = upload_mock.mock_calls[0][1][1][0]
|
||||
self.assertEqual(sys.stdin.fileno(), swift_upload_obj.source.fileno())
|
||||
io_open_mock.assert_called_once_with(sys.stdin.fileno(), mode='rb')
|
||||
|
||||
@mock.patch('swiftclient.service.SwiftService.upload')
|
||||
def test_upload_from_stdin_no_name(self, upload_mock):
|
||||
argv = ["", "upload", "container", "-"]
|
||||
with CaptureOutput() as out:
|
||||
self.assertRaises(SystemExit, swiftclient.shell.main, argv)
|
||||
self.assertEqual(0, len(upload_mock.mock_calls))
|
||||
self.assertTrue(out.err.find('object-name must be specified') >= 0)
|
||||
|
||||
@mock.patch('swiftclient.service.SwiftService.upload')
|
||||
def test_upload_from_stdin_and_others(self, upload_mock):
|
||||
argv = ["", "upload", "container", "-", "foo", "--object-name", "bar"]
|
||||
with CaptureOutput() as out:
|
||||
self.assertRaises(SystemExit, swiftclient.shell.main, argv)
|
||||
self.assertEqual(0, len(upload_mock.mock_calls))
|
||||
self.assertTrue(out.err.find(
|
||||
'upload from stdin cannot be used') >= 0)
|
||||
|
||||
@mock.patch.object(swiftclient.service.SwiftService,
|
||||
'_bulk_delete_page_size', lambda *a: 0)
|
||||
@mock.patch('swiftclient.service.Connection')
|
||||
|
Loading…
x
Reference in New Issue
Block a user