Allow for uploads from standard input.

If "-" is passed in for the source, python-swiftclient will upload
the object by reading the contents of the standard input. The object
name option must be set, as well, and this cannot be used in
conjunction with other files.

This approach stores the entire contents as one object. A follow on
patch will change this behavior to upload from standard input as SLO,
unless the segment size is larger than the content size.

Change-Id: I1a8be6377de06f702e0f336a5a593408ed49be02

doc/manpages/swift.1

@@ -63,8 +63,11 @@ Uploads to the given container the files and directories specified by the
 remaining args. The \-c or \-\-changed is an option that will only upload files
 that have changed since the last upload. The \-\-object\-name <object\-name> is
 an option that will upload file and name object to <object\-name> or upload dir
-and use <object\-name> as object prefix. The \-S <size> or \-\-segment\-size <size>
-and \-\-leave\-segments and others are options as well (see swift upload \-\-help for more).
+and use <object\-name> as object prefix. If the file name is "-", reads the
+content from standard input. In this case, \-\-object\-name is required and no
+other files may be given. The \-S <size> or \-\-segment\-size <size> and
+\-\-leave\-segments and others are options as well (see swift upload \-\-help
+for more).
 .RE
 
 \fBpost\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR]

doc/source/cli/index.rst

@@ -369,10 +369,10 @@ given container. The ``-c`` or ``--changed`` is an option that will only
 upload files that have changed since the last upload. The
 ``--object-name <object-name>`` is an option that will upload a file and
 name object to ``<object-name>`` or upload a directory and use ``<object-name>``
-as object prefix. The ``-S <size>`` or ``--segment-size <size>`` and
-``--leave-segments`` are options as well (see ``--help`` for more).
-
-Uploads specified files and directories to the given container.
+as object prefix. If the file name is "-", client reads content from standard
+input. In this case ``--object-name`` is required to set the name of the object
+and no other files may be given.  The ``-S <size>`` or ``--segment-size <size>``
+and ``--leave-segments`` are options as well (see ``--help`` for more).
 
 **Positional arguments:**
 

swiftclient/service.py

@@ -1811,6 +1811,8 @@ class SwiftService(object):
         return chunks
 
     def _is_identical(self, chunk_data, path):
+        if path is None:
+            return False
         try:
             fp = open(path, 'rb', DISK_BUFFER)
         except IOError:

swiftclient/shell.py

@@ -17,6 +17,7 @@
 from __future__ import print_function, unicode_literals
 
 import argparse
+import io
 import json
 import logging
 import signal
@@ -26,7 +27,7 @@ from os import environ, walk, _exit as os_exit
 from os.path import isfile, isdir, join
 from six import text_type, PY2
 from six.moves.urllib.parse import unquote, urlparse
-from sys import argv as sys_argv, exit, stderr
+from sys import argv as sys_argv, exit, stderr, stdin
 from time import gmtime, strftime
 
 from swiftclient import RequestException
@@ -901,7 +902,9 @@ Uploads specified files and directories to the given container.
 Positional arguments:
   <container>           Name of container to upload to.
   <file_or_directory>   Name of file or directory to upload. Specify multiple
-                        times for multiple uploads.
+                        times for multiple uploads. If "-" is specified, reads
+                        content from standard input (--object-name is required
+                        in this case).
 
 Optional arguments:
   -c, --changed         Only upload files that have changed since the last
@@ -1002,6 +1005,11 @@ def st_upload(parser, args, output_manager):
     else:
         container = args[0]
         files = args[1:]
+        from_stdin = '-' in files
+        if from_stdin and len(files) > 1:
+            output_manager.error(
+                'upload from stdin cannot be used along with other files')
+            return
 
     if options['object_name'] is not None:
         if len(files) > 1:
@@ -1009,6 +1017,10 @@ def st_upload(parser, args, output_manager):
             return
         else:
             orig_path = files[0]
+    elif from_stdin:
+        output_manager.error(
+            'object-name must be specified with uploads from stdin')
+        return
 
     if options['segment_size']:
         try:
@@ -1047,6 +1059,14 @@ def st_upload(parser, args, output_manager):
             objs = []
             dir_markers = []
             for f in files:
+                if f == '-':
+                    fd = io.open(stdin.fileno(), mode='rb')
+                    objs.append(SwiftUploadObject(
+                        fd, object_name=options['object_name']))
+                    # We ensure that there is exactly one "file" to upload in
+                    # this case -- stdin
+                    break
+
                 if isfile(f):
                     objs.append(f)
                 elif isdir(f):
@@ -1060,7 +1080,7 @@ def st_upload(parser, args, output_manager):
 
             # Now that we've collected all the required files and dir markers
             # build the tuples for the call to upload
-            if options['object_name'] is not None:
+            if options['object_name'] is not None and not from_stdin:
                 objs = [
                     SwiftUploadObject(
                         o, object_name=o.replace(

tests/unit/test_shell.py

@@ -27,6 +27,7 @@ from time import localtime, mktime, strftime, strptime
 
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 import six
+import sys
 
 import swiftclient
 from swiftclient.service import SwiftError
@@ -719,7 +720,7 @@ class TestShell(unittest.TestCase):
                 'x-object-meta-mtime': mock.ANY,
             },
             query_string='multipart-manifest=put',
-            response_dict={})
+            response_dict=mock.ANY)
 
     @mock.patch('swiftclient.service.SwiftService.upload')
     def test_upload_object_with_account_readonly(self, upload):
@@ -905,6 +906,44 @@ class TestShell(unittest.TestCase):
                      'x-object-meta-mtime': mock.ANY},
             response_dict={})
 
+    @mock.patch('swiftclient.shell.io.open')
+    @mock.patch('swiftclient.service.SwiftService.upload')
+    def test_upload_from_stdin(self, upload_mock, io_open_mock):
+        def fake_open(fd, mode):
+            mock_io = mock.Mock()
+            mock_io.fileno.return_value = fd
+            return mock_io
+
+        io_open_mock.side_effect = fake_open
+
+        argv = ["", "upload", "container", "-", "--object-name", "foo"]
+        swiftclient.shell.main(argv)
+        upload_mock.assert_called_once_with("container", mock.ANY)
+        # This is a little convoluted: we want to examine the first call ([0]),
+        # the argv list([1]), the second parameter ([1]), and the first
+        # element.  This is because the upload method takes a container and a
+        # list of SwiftUploadObjects.
+        swift_upload_obj = upload_mock.mock_calls[0][1][1][0]
+        self.assertEqual(sys.stdin.fileno(), swift_upload_obj.source.fileno())
+        io_open_mock.assert_called_once_with(sys.stdin.fileno(), mode='rb')
+
+    @mock.patch('swiftclient.service.SwiftService.upload')
+    def test_upload_from_stdin_no_name(self, upload_mock):
+        argv = ["", "upload", "container", "-"]
+        with CaptureOutput() as out:
+            self.assertRaises(SystemExit, swiftclient.shell.main, argv)
+            self.assertEqual(0, len(upload_mock.mock_calls))
+            self.assertTrue(out.err.find('object-name must be specified') >= 0)
+
+    @mock.patch('swiftclient.service.SwiftService.upload')
+    def test_upload_from_stdin_and_others(self, upload_mock):
+        argv = ["", "upload", "container", "-", "foo", "--object-name", "bar"]
+        with CaptureOutput() as out:
+            self.assertRaises(SystemExit, swiftclient.shell.main, argv)
+            self.assertEqual(0, len(upload_mock.mock_calls))
+            self.assertTrue(out.err.find(
+                'upload from stdin cannot be used') >= 0)
+
     @mock.patch.object(swiftclient.service.SwiftService,
                        '_bulk_delete_page_size', lambda *a: 0)
     @mock.patch('swiftclient.service.Connection')