From 0982791db2ccb851f277ffa653065e4021e52b3f Mon Sep 17 00:00:00 2001 From: Timur Alperovich Date: Thu, 15 Jun 2017 20:53:04 -0700 Subject: [PATCH] Allow for uploads from standard input. If "-" is passed in for the source, python-swiftclient will upload the object by reading the contents of the standard input. The object name option must be set, as well, and this cannot be used in conjunction with other files. This approach stores the entire contents as one object. A follow on patch will change this behavior to upload from standard input as SLO, unless the segment size is larger than the content size. Change-Id: I1a8be6377de06f702e0f336a5a593408ed49be02 --- doc/manpages/swift.1 | 7 +++++-- doc/source/cli/index.rst | 8 ++++---- swiftclient/service.py | 2 ++ swiftclient/shell.py | 26 ++++++++++++++++++++++--- tests/unit/test_shell.py | 41 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 74 insertions(+), 10 deletions(-) diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index f7723826..00e1440e 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -63,8 +63,11 @@ Uploads to the given container the files and directories specified by the remaining args. The \-c or \-\-changed is an option that will only upload files that have changed since the last upload. The \-\-object\-name is an option that will upload file and name object to or upload dir -and use as object prefix. The \-S or \-\-segment\-size -and \-\-leave\-segments and others are options as well (see swift upload \-\-help for more). +and use as object prefix. If the file name is "-", reads the +content from standard input. In this case, \-\-object\-name is required and no +other files may be given. The \-S or \-\-segment\-size and +\-\-leave\-segments and others are options as well (see swift upload \-\-help +for more). .RE \fBpost\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst index bded3497..bec1f5e5 100644 --- a/doc/source/cli/index.rst +++ b/doc/source/cli/index.rst @@ -369,10 +369,10 @@ given container. The ``-c`` or ``--changed`` is an option that will only upload files that have changed since the last upload. The ``--object-name `` is an option that will upload a file and name object to ```` or upload a directory and use ```` -as object prefix. The ``-S `` or ``--segment-size `` and -``--leave-segments`` are options as well (see ``--help`` for more). - -Uploads specified files and directories to the given container. +as object prefix. If the file name is "-", client reads content from standard +input. In this case ``--object-name`` is required to set the name of the object +and no other files may be given. The ``-S `` or ``--segment-size `` +and ``--leave-segments`` are options as well (see ``--help`` for more). **Positional arguments:** diff --git a/swiftclient/service.py b/swiftclient/service.py index 5f032be7..31ea898e 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1811,6 +1811,8 @@ class SwiftService(object): return chunks def _is_identical(self, chunk_data, path): + if path is None: + return False try: fp = open(path, 'rb', DISK_BUFFER) except IOError: diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 19a224a2..894cd291 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -17,6 +17,7 @@ from __future__ import print_function, unicode_literals import argparse +import io import json import logging import signal @@ -26,7 +27,7 @@ from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join from six import text_type, PY2 from six.moves.urllib.parse import unquote, urlparse -from sys import argv as sys_argv, exit, stderr +from sys import argv as sys_argv, exit, stderr, stdin from time import gmtime, strftime from swiftclient import RequestException @@ -901,7 +902,9 @@ Uploads specified files and directories to the given container. Positional arguments: Name of container to upload to. Name of file or directory to upload. Specify multiple - times for multiple uploads. + times for multiple uploads. If "-" is specified, reads + content from standard input (--object-name is required + in this case). Optional arguments: -c, --changed Only upload files that have changed since the last @@ -1002,6 +1005,11 @@ def st_upload(parser, args, output_manager): else: container = args[0] files = args[1:] + from_stdin = '-' in files + if from_stdin and len(files) > 1: + output_manager.error( + 'upload from stdin cannot be used along with other files') + return if options['object_name'] is not None: if len(files) > 1: @@ -1009,6 +1017,10 @@ def st_upload(parser, args, output_manager): return else: orig_path = files[0] + elif from_stdin: + output_manager.error( + 'object-name must be specified with uploads from stdin') + return if options['segment_size']: try: @@ -1047,6 +1059,14 @@ def st_upload(parser, args, output_manager): objs = [] dir_markers = [] for f in files: + if f == '-': + fd = io.open(stdin.fileno(), mode='rb') + objs.append(SwiftUploadObject( + fd, object_name=options['object_name'])) + # We ensure that there is exactly one "file" to upload in + # this case -- stdin + break + if isfile(f): objs.append(f) elif isdir(f): @@ -1060,7 +1080,7 @@ def st_upload(parser, args, output_manager): # Now that we've collected all the required files and dir markers # build the tuples for the call to upload - if options['object_name'] is not None: + if options['object_name'] is not None and not from_stdin: objs = [ SwiftUploadObject( o, object_name=o.replace( diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 4c47db14..2e20a876 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -27,6 +27,7 @@ from time import localtime, mktime, strftime, strptime from requests.packages.urllib3.exceptions import InsecureRequestWarning import six +import sys import swiftclient from swiftclient.service import SwiftError @@ -719,7 +720,7 @@ class TestShell(unittest.TestCase): 'x-object-meta-mtime': mock.ANY, }, query_string='multipart-manifest=put', - response_dict={}) + response_dict=mock.ANY) @mock.patch('swiftclient.service.SwiftService.upload') def test_upload_object_with_account_readonly(self, upload): @@ -905,6 +906,44 @@ class TestShell(unittest.TestCase): 'x-object-meta-mtime': mock.ANY}, response_dict={}) + @mock.patch('swiftclient.shell.io.open') + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin(self, upload_mock, io_open_mock): + def fake_open(fd, mode): + mock_io = mock.Mock() + mock_io.fileno.return_value = fd + return mock_io + + io_open_mock.side_effect = fake_open + + argv = ["", "upload", "container", "-", "--object-name", "foo"] + swiftclient.shell.main(argv) + upload_mock.assert_called_once_with("container", mock.ANY) + # This is a little convoluted: we want to examine the first call ([0]), + # the argv list([1]), the second parameter ([1]), and the first + # element. This is because the upload method takes a container and a + # list of SwiftUploadObjects. + swift_upload_obj = upload_mock.mock_calls[0][1][1][0] + self.assertEqual(sys.stdin.fileno(), swift_upload_obj.source.fileno()) + io_open_mock.assert_called_once_with(sys.stdin.fileno(), mode='rb') + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin_no_name(self, upload_mock): + argv = ["", "upload", "container", "-"] + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(0, len(upload_mock.mock_calls)) + self.assertTrue(out.err.find('object-name must be specified') >= 0) + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin_and_others(self, upload_mock): + argv = ["", "upload", "container", "-", "foo", "--object-name", "bar"] + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(0, len(upload_mock.mock_calls)) + self.assertTrue(out.err.find( + 'upload from stdin cannot be used') >= 0) + @mock.patch.object(swiftclient.service.SwiftService, '_bulk_delete_page_size', lambda *a: 0) @mock.patch('swiftclient.service.Connection')