Allow for uploads from standard input.

If "-" is passed in for the source, python-swiftclient will upload
the object by reading the contents of the standard input. The object
name option must be set, as well, and this cannot be used in
conjunction with other files.

This approach stores the entire contents as one object. A follow on
patch will change this behavior to upload from standard input as SLO,
unless the segment size is larger than the content size.

Change-Id: I1a8be6377de06f702e0f336a5a593408ed49be02
This commit is contained in:
Timur Alperovich 2017-06-15 20:53:04 -07:00
parent 124c7de676
commit 0982791db2
5 changed files with 74 additions and 10 deletions

View File

@ -63,8 +63,11 @@ Uploads to the given container the files and directories specified by the
remaining args. The \-c or \-\-changed is an option that will only upload files
that have changed since the last upload. The \-\-object\-name <object\-name> is
an option that will upload file and name object to <object\-name> or upload dir
and use <object\-name> as object prefix. The \-S <size> or \-\-segment\-size <size>
and \-\-leave\-segments and others are options as well (see swift upload \-\-help for more).
and use <object\-name> as object prefix. If the file name is "-", reads the
content from standard input. In this case, \-\-object\-name is required and no
other files may be given. The \-S <size> or \-\-segment\-size <size> and
\-\-leave\-segments and others are options as well (see swift upload \-\-help
for more).
.RE
\fBpost\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR]

View File

@ -369,10 +369,10 @@ given container. The ``-c`` or ``--changed`` is an option that will only
upload files that have changed since the last upload. The
``--object-name <object-name>`` is an option that will upload a file and
name object to ``<object-name>`` or upload a directory and use ``<object-name>``
as object prefix. The ``-S <size>`` or ``--segment-size <size>`` and
``--leave-segments`` are options as well (see ``--help`` for more).
Uploads specified files and directories to the given container.
as object prefix. If the file name is "-", client reads content from standard
input. In this case ``--object-name`` is required to set the name of the object
and no other files may be given. The ``-S <size>`` or ``--segment-size <size>``
and ``--leave-segments`` are options as well (see ``--help`` for more).
**Positional arguments:**

View File

@ -1811,6 +1811,8 @@ class SwiftService(object):
return chunks
def _is_identical(self, chunk_data, path):
if path is None:
return False
try:
fp = open(path, 'rb', DISK_BUFFER)
except IOError:

View File

@ -17,6 +17,7 @@
from __future__ import print_function, unicode_literals
import argparse
import io
import json
import logging
import signal
@ -26,7 +27,7 @@ from os import environ, walk, _exit as os_exit
from os.path import isfile, isdir, join
from six import text_type, PY2
from six.moves.urllib.parse import unquote, urlparse
from sys import argv as sys_argv, exit, stderr
from sys import argv as sys_argv, exit, stderr, stdin
from time import gmtime, strftime
from swiftclient import RequestException
@ -901,7 +902,9 @@ Uploads specified files and directories to the given container.
Positional arguments:
<container> Name of container to upload to.
<file_or_directory> Name of file or directory to upload. Specify multiple
times for multiple uploads.
times for multiple uploads. If "-" is specified, reads
content from standard input (--object-name is required
in this case).
Optional arguments:
-c, --changed Only upload files that have changed since the last
@ -1002,6 +1005,11 @@ def st_upload(parser, args, output_manager):
else:
container = args[0]
files = args[1:]
from_stdin = '-' in files
if from_stdin and len(files) > 1:
output_manager.error(
'upload from stdin cannot be used along with other files')
return
if options['object_name'] is not None:
if len(files) > 1:
@ -1009,6 +1017,10 @@ def st_upload(parser, args, output_manager):
return
else:
orig_path = files[0]
elif from_stdin:
output_manager.error(
'object-name must be specified with uploads from stdin')
return
if options['segment_size']:
try:
@ -1047,6 +1059,14 @@ def st_upload(parser, args, output_manager):
objs = []
dir_markers = []
for f in files:
if f == '-':
fd = io.open(stdin.fileno(), mode='rb')
objs.append(SwiftUploadObject(
fd, object_name=options['object_name']))
# We ensure that there is exactly one "file" to upload in
# this case -- stdin
break
if isfile(f):
objs.append(f)
elif isdir(f):
@ -1060,7 +1080,7 @@ def st_upload(parser, args, output_manager):
# Now that we've collected all the required files and dir markers
# build the tuples for the call to upload
if options['object_name'] is not None:
if options['object_name'] is not None and not from_stdin:
objs = [
SwiftUploadObject(
o, object_name=o.replace(

View File

@ -27,6 +27,7 @@ from time import localtime, mktime, strftime, strptime
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import six
import sys
import swiftclient
from swiftclient.service import SwiftError
@ -719,7 +720,7 @@ class TestShell(unittest.TestCase):
'x-object-meta-mtime': mock.ANY,
},
query_string='multipart-manifest=put',
response_dict={})
response_dict=mock.ANY)
@mock.patch('swiftclient.service.SwiftService.upload')
def test_upload_object_with_account_readonly(self, upload):
@ -905,6 +906,44 @@ class TestShell(unittest.TestCase):
'x-object-meta-mtime': mock.ANY},
response_dict={})
@mock.patch('swiftclient.shell.io.open')
@mock.patch('swiftclient.service.SwiftService.upload')
def test_upload_from_stdin(self, upload_mock, io_open_mock):
def fake_open(fd, mode):
mock_io = mock.Mock()
mock_io.fileno.return_value = fd
return mock_io
io_open_mock.side_effect = fake_open
argv = ["", "upload", "container", "-", "--object-name", "foo"]
swiftclient.shell.main(argv)
upload_mock.assert_called_once_with("container", mock.ANY)
# This is a little convoluted: we want to examine the first call ([0]),
# the argv list([1]), the second parameter ([1]), and the first
# element. This is because the upload method takes a container and a
# list of SwiftUploadObjects.
swift_upload_obj = upload_mock.mock_calls[0][1][1][0]
self.assertEqual(sys.stdin.fileno(), swift_upload_obj.source.fileno())
io_open_mock.assert_called_once_with(sys.stdin.fileno(), mode='rb')
@mock.patch('swiftclient.service.SwiftService.upload')
def test_upload_from_stdin_no_name(self, upload_mock):
argv = ["", "upload", "container", "-"]
with CaptureOutput() as out:
self.assertRaises(SystemExit, swiftclient.shell.main, argv)
self.assertEqual(0, len(upload_mock.mock_calls))
self.assertTrue(out.err.find('object-name must be specified') >= 0)
@mock.patch('swiftclient.service.SwiftService.upload')
def test_upload_from_stdin_and_others(self, upload_mock):
argv = ["", "upload", "container", "-", "foo", "--object-name", "bar"]
with CaptureOutput() as out:
self.assertRaises(SystemExit, swiftclient.shell.main, argv)
self.assertEqual(0, len(upload_mock.mock_calls))
self.assertTrue(out.err.find(
'upload from stdin cannot be used') >= 0)
@mock.patch.object(swiftclient.service.SwiftService,
'_bulk_delete_page_size', lambda *a: 0)
@mock.patch('swiftclient.service.Connection')