Add ability to download objects to particular folder.

This patch adds "--output-dir" and "--remove-prefix" options
to the "download" command and unit tests for it.

Example:

    $ swift list example --prefix swift2.2
    swift2.2/bin/swift-object-auditor
    swift2.2/bin/swift-object-expirer
    swift2.2/bin/swift-object-info
    swift2.2/bin/swift-object-replicator
    swift2.2/bin/swift-object-server
    swift2.2/bin/swift-object-updater

When given "--output-dir <directory>", client downloads objects to <directory>.

    $ swift download example --prefix swift2.2 \
                    --output-dir new/swift/dir

    The folder structure:
    .
    └── new
        └── swift
            └── dir
                └── swift2.2
                    └── bin
                        ├── swift-object-auditor
                        ├── swift-object-expirer
                        ├── swift-object-info
                        ├── swift-object-replicator
                        ├── swift-object-server
                        └── swift-object-updater

When given "--remove-prefix", client downloads objects without <prefix>.

    $ swift download example --prefix swift2.2 \
                    --remove-prefix \
                    --output-dir swift

    The folder structure:
    .
    └── swift
        └── bin
            ├── swift-object-auditor
            ├── swift-object-expirer
            ├── swift-object-info
            ├── swift-object-replicator
            ├── swift-object-server
            └── swift-object-updater

Co-Authored-By: Clay Gerrard <clay.gerrard@gmail.com>
Change-Id: I7463fe2941cc94f9a50a4756a97c2ccdf946294d
Implements: blueprint swiftclient-download-pseudo-folder-to-specific-target

swiftclient/service.py

@@ -12,6 +12,7 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
 from concurrent.futures import as_completed, CancelledError, TimeoutError
 from copy import deepcopy
 from errno import EEXIST, ENOENT
@@ -162,6 +163,8 @@ _default_local_options = {
     'read_acl': None,
     'write_acl': None,
     'out_file': None,
+    'out_directory': None,
+    'remove_prefix': False,
     'no_download': False,
     'long': False,
     'totals': False,
@@ -889,7 +892,9 @@ class SwiftService(object):
                                 'no_download': False,
                                 'header': [],
                                 'skip_identical': False,
-                                'out_file': None
+                                'out_directory': None,
+                                'out_file': None,
+                                'remove_prefix': False,
                             }
 
         :returns: A generator for returning the results of the download
@@ -986,6 +991,12 @@ class SwiftService(object):
         options['skip_identical'] = (options['skip_identical'] and
                                      out_file != '-')
 
+        if options['prefix'] and options['remove_prefix']:
+            path = path[len(options['prefix']):].lstrip('/')
+
+        if options['out_directory']:
+            path = os.path.join(options['out_directory'], path)
+
         if options['skip_identical']:
             filename = out_file if out_file else path
             try:

swiftclient/shell.py

@@ -146,9 +146,11 @@ def st_delete(parser, args, output_manager):
 
 
 st_download_options = '''[--all] [--marker] [--prefix <prefix>]
-                      [--output <out_file>] [--object-threads <threads>]
+                      [--output <out_file>] [--output-dir <out_directory>]
+                      [--object-threads <threads>]
                       [--container-threads <threads>] [--no-download]
-                      [--skip-identical] <container> <object>
+                      [--skip-identical] [--remove-prefix]
+                      <container> <object>
 '''
 
 st_download_help = '''
@@ -167,9 +169,15 @@ Optional arguments:
   --marker              Marker to use when starting a container or account
                         download.
   --prefix <prefix>     Only download items beginning with <prefix>
+  --remove-prefix       An optional flag for --prefix <prefix>, use this
+                        option to download items without <prefix>
   --output <out_file>   For a single file download, stream the output to
                         <out_file>. Specifying "-" as <out_file> will
                         redirect to stdout.
+  --output-dir <out_directory>
+                        An optional directory to which to store objects.
+                        By default, all objects are recreated in the current
+                        directory.
   --object-threads <threads>
                         Number of threads to use for downloading objects.
                         Default is 10.
@@ -203,6 +211,14 @@ def st_download(parser, args, output_manager):
         '-o', '--output', dest='out_file', help='For a single '
         'download, stream the output to <out_file>. '
         'Specifying "-" as <out_file> will redirect to stdout.')
+    parser.add_option(
+        '-D', '--output-dir', dest='out_directory',
+        help='An optional directory to which to store objects. '
+        'By default, all objects are recreated in the current directory.')
+    parser.add_option(
+        '-r', '--remove-prefix', action='store_true', dest='remove_prefix',
+        default=False, help='An optional flag for --prefix <prefix>, '
+        'use this option to download items without <prefix>.')
     parser.add_option(
         '', '--object-threads', type=int,
         default=10, help='Number of threads to use for downloading objects. '
@@ -233,6 +249,12 @@ def st_download(parser, args, output_manager):
     if options.out_file and len(args) != 2:
         exit('-o option only allowed for single file downloads')
 
+    if not options.prefix:
+        options.remove_prefix = False
+
+    if options.out_directory and len(args) == 2:
+        exit('Please use -o option for single file downloads and renames')
+
     if (not args and not options.yes_all) or (args and options.yes_all):
         output_manager.error('Usage: %s download %s\n%s', BASENAME,
                              st_download_options, st_download_help)

tests/unit/test_service.py

@@ -992,6 +992,17 @@ class TestServiceUpload(testtools.TestCase):
 
 class TestServiceDownload(testtools.TestCase):
 
+    def setUp(self):
+        super(TestServiceDownload, self).setUp()
+        self.opts = swiftclient.service._default_local_options.copy()
+        self.opts['no_download'] = True
+        self.obj_content = b'c' * 10
+        self.obj_etag = md5(self.obj_content).hexdigest()
+        self.obj_len = len(self.obj_content)
+
+    def _readbody(self):
+        yield self.obj_content
+
     def _assertDictEqual(self, a, b, m=None):
         # assertDictEqual is not available in py2.6 so use a shallow check
         # instead
@@ -1008,6 +1019,103 @@ class TestServiceDownload(testtools.TestCase):
                 self.assertIn(k, b, m)
                 self.assertEqual(b[k], v, m)
 
+    def test_download(self):
+        service = SwiftService()
+        with mock.patch('swiftclient.service.Connection') as mock_conn:
+            header = {'content-length': self.obj_len,
+                      'etag': self.obj_etag}
+            mock_conn.get_object.return_value = header, self._readbody()
+
+            resp = service._download_object_job(mock_conn,
+                                                'c',
+                                                'test',
+                                                self.opts)
+
+        self.assertTrue(resp['success'])
+        self.assertEqual(resp['action'], 'download_object')
+        self.assertEqual(resp['object'], 'test')
+        self.assertEqual(resp['path'], 'test')
+
+    def test_download_with_output_dir(self):
+        service = SwiftService()
+        with mock.patch('swiftclient.service.Connection') as mock_conn:
+            header = {'content-length': self.obj_len,
+                      'etag': self.obj_etag}
+            mock_conn.get_object.return_value = header, self._readbody()
+
+            options = self.opts.copy()
+            options['out_directory'] = 'temp_dir'
+            resp = service._download_object_job(mock_conn,
+                                                'c',
+                                                'example/test',
+                                                options)
+
+        self.assertTrue(resp['success'])
+        self.assertEqual(resp['action'], 'download_object')
+        self.assertEqual(resp['object'], 'example/test')
+        self.assertEqual(resp['path'], 'temp_dir/example/test')
+
+    def test_download_with_remove_prefix(self):
+        service = SwiftService()
+        with mock.patch('swiftclient.service.Connection') as mock_conn:
+            header = {'content-length': self.obj_len,
+                      'etag': self.obj_etag}
+            mock_conn.get_object.return_value = header, self._readbody()
+
+            options = self.opts.copy()
+            options['prefix'] = 'example/'
+            options['remove_prefix'] = True
+            resp = service._download_object_job(mock_conn,
+                                                'c',
+                                                'example/test',
+                                                options)
+
+        self.assertTrue(resp['success'])
+        self.assertEqual(resp['action'], 'download_object')
+        self.assertEqual(resp['object'], 'example/test')
+        self.assertEqual(resp['path'], 'test')
+
+    def test_download_with_remove_prefix_and_remove_slashes(self):
+        service = SwiftService()
+        with mock.patch('swiftclient.service.Connection') as mock_conn:
+            header = {'content-length': self.obj_len,
+                      'etag': self.obj_etag}
+            mock_conn.get_object.return_value = header, self._readbody()
+
+            options = self.opts.copy()
+            options['prefix'] = 'example'
+            options['remove_prefix'] = True
+            resp = service._download_object_job(mock_conn,
+                                                'c',
+                                                'example/test',
+                                                options)
+
+        self.assertTrue(resp['success'])
+        self.assertEqual(resp['action'], 'download_object')
+        self.assertEqual(resp['object'], 'example/test')
+        self.assertEqual(resp['path'], 'test')
+
+    def test_download_with_output_dir_and_remove_prefix(self):
+        service = SwiftService()
+        with mock.patch('swiftclient.service.Connection') as mock_conn:
+            header = {'content-length': self.obj_len,
+                      'etag': self.obj_etag}
+            mock_conn.get_object.return_value = header, self._readbody()
+
+            options = self.opts.copy()
+            options['prefix'] = 'example'
+            options['out_directory'] = 'new/dir'
+            options['remove_prefix'] = True
+            resp = service._download_object_job(mock_conn,
+                                                'c',
+                                                'example/test',
+                                                options)
+
+        self.assertTrue(resp['success'])
+        self.assertEqual(resp['action'], 'download_object')
+        self.assertEqual(resp['object'], 'example/test')
+        self.assertEqual(resp['path'], 'new/dir/test')
+
     def test_download_object_job_skip_identical(self):
         with tempfile.NamedTemporaryFile() as f:
             f.write(b'a' * 30)
@@ -1040,6 +1148,9 @@ class TestServiceDownload(testtools.TestCase):
                                        container='test_c',
                                        obj='test_o',
                                        options={'out_file': f.name,
+                                                'out_directory': None,
+                                                'prefix': None,
+                                                'remove_prefix': False,
                                                 'header': {},
                                                 'yes_all': False,
                                                 'skip_identical': True})
@@ -1092,6 +1203,9 @@ class TestServiceDownload(testtools.TestCase):
                                            container='test_c',
                                            obj='test_o',
                                            options={'out_file': f.name,
+                                                    'out_directory': None,
+                                                    'prefix': None,
+                                                    'remove_prefix': False,
                                                     'header': {},
                                                     'yes_all': False,
                                                     'skip_identical': True})
@@ -1170,6 +1284,9 @@ class TestServiceDownload(testtools.TestCase):
                                            container='test_c',
                                            obj='test_o',
                                            options={'out_file': f.name,
+                                                    'out_directory': None,
+                                                    'prefix': None,
+                                                    'remove_prefix': False,
                                                     'header': {},
                                                     'yes_all': False,
                                                     'skip_identical': True})
@@ -1231,6 +1348,9 @@ class TestServiceDownload(testtools.TestCase):
                 'auth_end_time': mock_conn.auth_end_time,
             }
 
+            options = self.opts.copy()
+            options['out_file'] = f.name
+            options['skip_identical'] = True
             s = SwiftService()
             with mock.patch('swiftclient.service.time', side_effect=range(3)):
                 with mock.patch('swiftclient.service.get_conn',
@@ -1239,11 +1359,7 @@ class TestServiceDownload(testtools.TestCase):
                         conn=mock_conn,
                         container='test_c',
                         obj='test_o',
-                        options={'out_file': f.name,
+                        options=options)
-                                 'header': {},
-                                 'no_download': True,
-                                 'yes_all': False,
-                                 'skip_identical': True})
 
             self._assertDictEqual(r, expected_r)
 
@@ -1323,6 +1439,9 @@ class TestServiceDownload(testtools.TestCase):
                 'auth_end_time': mock_conn.auth_end_time,
             }
 
+            options = self.opts.copy()
+            options['out_file'] = f.name
+            options['skip_identical'] = True
             s = SwiftService()
             with mock.patch('swiftclient.service.time', side_effect=range(3)):
                 with mock.patch('swiftclient.service.get_conn',
@@ -1331,11 +1450,7 @@ class TestServiceDownload(testtools.TestCase):
                         conn=mock_conn,
                         container='test_c',
                         obj='test_o',
-                        options={'out_file': f.name,
+                        options=options)
-                                 'header': {},
-                                 'no_download': True,
-                                 'yes_all': False,
-                                 'skip_identical': True})
 
             self._assertDictEqual(r, expected_r)
             self.assertEqual(mock_conn.get_object.mock_calls, [