Add ability to download objects to particular folder.
This patch adds "--output-dir" and "--remove-prefix" options
to the "download" command and unit tests for it.
Example:
$ swift list example --prefix swift2.2
swift2.2/bin/swift-object-auditor
swift2.2/bin/swift-object-expirer
swift2.2/bin/swift-object-info
swift2.2/bin/swift-object-replicator
swift2.2/bin/swift-object-server
swift2.2/bin/swift-object-updater
When given "--output-dir <directory>", client downloads objects to <directory>.
$ swift download example --prefix swift2.2 \
--output-dir new/swift/dir
The folder structure:
.
└── new
└── swift
└── dir
└── swift2.2
└── bin
├── swift-object-auditor
├── swift-object-expirer
├── swift-object-info
├── swift-object-replicator
├── swift-object-server
└── swift-object-updater
When given "--remove-prefix", client downloads objects without <prefix>.
$ swift download example --prefix swift2.2 \
--remove-prefix \
--output-dir swift
The folder structure:
.
└── swift
└── bin
├── swift-object-auditor
├── swift-object-expirer
├── swift-object-info
├── swift-object-replicator
├── swift-object-server
└── swift-object-updater
Co-Authored-By: Clay Gerrard <clay.gerrard@gmail.com>
Change-Id: I7463fe2941cc94f9a50a4756a97c2ccdf946294d
Implements: blueprint swiftclient-download-pseudo-folder-to-specific-target
This commit is contained in:
Charles Hsu
parent
ec3e2ab3a0
commit
d5d3127744
@ -12,6 +12,7 @@
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
from concurrent.futures import as_completed, CancelledError, TimeoutError
|
||||
from copy import deepcopy
|
||||
from errno import EEXIST, ENOENT
|
||||
@ -162,6 +163,8 @@ _default_local_options = {
|
||||
'read_acl': None,
|
||||
'write_acl': None,
|
||||
'out_file': None,
|
||||
'out_directory': None,
|
||||
'remove_prefix': False,
|
||||
'no_download': False,
|
||||
'long': False,
|
||||
'totals': False,
|
||||
@ -889,7 +892,9 @@ class SwiftService(object):
|
||||
'no_download': False,
|
||||
'header': [],
|
||||
'skip_identical': False,
|
||||
'out_file': None
|
||||
'out_directory': None,
|
||||
'out_file': None,
|
||||
'remove_prefix': False,
|
||||
}
|
||||
|
||||
:returns: A generator for returning the results of the download
|
||||
@ -986,6 +991,12 @@ class SwiftService(object):
|
||||
options['skip_identical'] = (options['skip_identical'] and
|
||||
out_file != '-')
|
||||
|
||||
if options['prefix'] and options['remove_prefix']:
|
||||
path = path[len(options['prefix']):].lstrip('/')
|
||||
|
||||
if options['out_directory']:
|
||||
path = os.path.join(options['out_directory'], path)
|
||||
|
||||
if options['skip_identical']:
|
||||
filename = out_file if out_file else path
|
||||
try:
|
||||
|
||||
@ -146,9 +146,11 @@ def st_delete(parser, args, output_manager):
|
||||
|
||||
|
||||
st_download_options = '''[--all] [--marker] [--prefix <prefix>]
|
||||
[--output <out_file>] [--object-threads <threads>]
|
||||
[--output <out_file>] [--output-dir <out_directory>]
|
||||
[--object-threads <threads>]
|
||||
[--container-threads <threads>] [--no-download]
|
||||
[--skip-identical] <container> <object>
|
||||
[--skip-identical] [--remove-prefix]
|
||||
<container> <object>
|
||||
'''
|
||||
|
||||
st_download_help = '''
|
||||
@ -167,9 +169,15 @@ Optional arguments:
|
||||
--marker Marker to use when starting a container or account
|
||||
download.
|
||||
--prefix <prefix> Only download items beginning with <prefix>
|
||||
--remove-prefix An optional flag for --prefix <prefix>, use this
|
||||
option to download items without <prefix>
|
||||
--output <out_file> For a single file download, stream the output to
|
||||
<out_file>. Specifying "-" as <out_file> will
|
||||
redirect to stdout.
|
||||
--output-dir <out_directory>
|
||||
An optional directory to which to store objects.
|
||||
By default, all objects are recreated in the current
|
||||
directory.
|
||||
--object-threads <threads>
|
||||
Number of threads to use for downloading objects.
|
||||
Default is 10.
|
||||
@ -203,6 +211,14 @@ def st_download(parser, args, output_manager):
|
||||
'-o', '--output', dest='out_file', help='For a single '
|
||||
'download, stream the output to <out_file>. '
|
||||
'Specifying "-" as <out_file> will redirect to stdout.')
|
||||
parser.add_option(
|
||||
'-D', '--output-dir', dest='out_directory',
|
||||
help='An optional directory to which to store objects. '
|
||||
'By default, all objects are recreated in the current directory.')
|
||||
parser.add_option(
|
||||
'-r', '--remove-prefix', action='store_true', dest='remove_prefix',
|
||||
default=False, help='An optional flag for --prefix <prefix>, '
|
||||
'use this option to download items without <prefix>.')
|
||||
parser.add_option(
|
||||
'', '--object-threads', type=int,
|
||||
default=10, help='Number of threads to use for downloading objects. '
|
||||
@ -233,6 +249,12 @@ def st_download(parser, args, output_manager):
|
||||
if options.out_file and len(args) != 2:
|
||||
exit('-o option only allowed for single file downloads')
|
||||
|
||||
if not options.prefix:
|
||||
options.remove_prefix = False
|
||||
|
||||
if options.out_directory and len(args) == 2:
|
||||
exit('Please use -o option for single file downloads and renames')
|
||||
|
||||
if (not args and not options.yes_all) or (args and options.yes_all):
|
||||
output_manager.error('Usage: %s download %s\n%s', BASENAME,
|
||||
st_download_options, st_download_help)
|
||||
|
||||
@ -992,6 +992,17 @@ class TestServiceUpload(testtools.TestCase):
|
||||
|
||||
class TestServiceDownload(testtools.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestServiceDownload, self).setUp()
|
||||
self.opts = swiftclient.service._default_local_options.copy()
|
||||
self.opts['no_download'] = True
|
||||
self.obj_content = b'c' * 10
|
||||
self.obj_etag = md5(self.obj_content).hexdigest()
|
||||
self.obj_len = len(self.obj_content)
|
||||
|
||||
def _readbody(self):
|
||||
yield self.obj_content
|
||||
|
||||
def _assertDictEqual(self, a, b, m=None):
|
||||
# assertDictEqual is not available in py2.6 so use a shallow check
|
||||
# instead
|
||||
@ -1008,6 +1019,103 @@ class TestServiceDownload(testtools.TestCase):
|
||||
self.assertIn(k, b, m)
|
||||
self.assertEqual(b[k], v, m)
|
||||
|
||||
def test_download(self):
|
||||
service = SwiftService()
|
||||
with mock.patch('swiftclient.service.Connection') as mock_conn:
|
||||
header = {'content-length': self.obj_len,
|
||||
'etag': self.obj_etag}
|
||||
mock_conn.get_object.return_value = header, self._readbody()
|
||||
|
||||
resp = service._download_object_job(mock_conn,
|
||||
'c',
|
||||
'test',
|
||||
self.opts)
|
||||
|
||||
self.assertTrue(resp['success'])
|
||||
self.assertEqual(resp['action'], 'download_object')
|
||||
self.assertEqual(resp['object'], 'test')
|
||||
self.assertEqual(resp['path'], 'test')
|
||||
|
||||
def test_download_with_output_dir(self):
|
||||
service = SwiftService()
|
||||
with mock.patch('swiftclient.service.Connection') as mock_conn:
|
||||
header = {'content-length': self.obj_len,
|
||||
'etag': self.obj_etag}
|
||||
mock_conn.get_object.return_value = header, self._readbody()
|
||||
|
||||
options = self.opts.copy()
|
||||
options['out_directory'] = 'temp_dir'
|
||||
resp = service._download_object_job(mock_conn,
|
||||
'c',
|
||||
'example/test',
|
||||
options)
|
||||
|
||||
self.assertTrue(resp['success'])
|
||||
self.assertEqual(resp['action'], 'download_object')
|
||||
self.assertEqual(resp['object'], 'example/test')
|
||||
self.assertEqual(resp['path'], 'temp_dir/example/test')
|
||||
|
||||
def test_download_with_remove_prefix(self):
|
||||
service = SwiftService()
|
||||
with mock.patch('swiftclient.service.Connection') as mock_conn:
|
||||
header = {'content-length': self.obj_len,
|
||||
'etag': self.obj_etag}
|
||||
mock_conn.get_object.return_value = header, self._readbody()
|
||||
|
||||
options = self.opts.copy()
|
||||
options['prefix'] = 'example/'
|
||||
options['remove_prefix'] = True
|
||||
resp = service._download_object_job(mock_conn,
|
||||
'c',
|
||||
'example/test',
|
||||
options)
|
||||
|
||||
self.assertTrue(resp['success'])
|
||||
self.assertEqual(resp['action'], 'download_object')
|
||||
self.assertEqual(resp['object'], 'example/test')
|
||||
self.assertEqual(resp['path'], 'test')
|
||||
|
||||
def test_download_with_remove_prefix_and_remove_slashes(self):
|
||||
service = SwiftService()
|
||||
with mock.patch('swiftclient.service.Connection') as mock_conn:
|
||||
header = {'content-length': self.obj_len,
|
||||
'etag': self.obj_etag}
|
||||
mock_conn.get_object.return_value = header, self._readbody()
|
||||
|
||||
options = self.opts.copy()
|
||||
options['prefix'] = 'example'
|
||||
options['remove_prefix'] = True
|
||||
resp = service._download_object_job(mock_conn,
|
||||
'c',
|
||||
'example/test',
|
||||
options)
|
||||
|
||||
self.assertTrue(resp['success'])
|
||||
self.assertEqual(resp['action'], 'download_object')
|
||||
self.assertEqual(resp['object'], 'example/test')
|
||||
self.assertEqual(resp['path'], 'test')
|
||||
|
||||
def test_download_with_output_dir_and_remove_prefix(self):
|
||||
service = SwiftService()
|
||||
with mock.patch('swiftclient.service.Connection') as mock_conn:
|
||||
header = {'content-length': self.obj_len,
|
||||
'etag': self.obj_etag}
|
||||
mock_conn.get_object.return_value = header, self._readbody()
|
||||
|
||||
options = self.opts.copy()
|
||||
options['prefix'] = 'example'
|
||||
options['out_directory'] = 'new/dir'
|
||||
options['remove_prefix'] = True
|
||||
resp = service._download_object_job(mock_conn,
|
||||
'c',
|
||||
'example/test',
|
||||
options)
|
||||
|
||||
self.assertTrue(resp['success'])
|
||||
self.assertEqual(resp['action'], 'download_object')
|
||||
self.assertEqual(resp['object'], 'example/test')
|
||||
self.assertEqual(resp['path'], 'new/dir/test')
|
||||
|
||||
def test_download_object_job_skip_identical(self):
|
||||
with tempfile.NamedTemporaryFile() as f:
|
||||
f.write(b'a' * 30)
|
||||
@ -1040,6 +1148,9 @@ class TestServiceDownload(testtools.TestCase):
|
||||
container='test_c',
|
||||
obj='test_o',
|
||||
options={'out_file': f.name,
|
||||
'out_directory': None,
|
||||
'prefix': None,
|
||||
'remove_prefix': False,
|
||||
'header': {},
|
||||
'yes_all': False,
|
||||
'skip_identical': True})
|
||||
@ -1092,6 +1203,9 @@ class TestServiceDownload(testtools.TestCase):
|
||||
container='test_c',
|
||||
obj='test_o',
|
||||
options={'out_file': f.name,
|
||||
'out_directory': None,
|
||||
'prefix': None,
|
||||
'remove_prefix': False,
|
||||
'header': {},
|
||||
'yes_all': False,
|
||||
'skip_identical': True})
|
||||
@ -1170,6 +1284,9 @@ class TestServiceDownload(testtools.TestCase):
|
||||
container='test_c',
|
||||
obj='test_o',
|
||||
options={'out_file': f.name,
|
||||
'out_directory': None,
|
||||
'prefix': None,
|
||||
'remove_prefix': False,
|
||||
'header': {},
|
||||
'yes_all': False,
|
||||
'skip_identical': True})
|
||||
@ -1231,6 +1348,9 @@ class TestServiceDownload(testtools.TestCase):
|
||||
'auth_end_time': mock_conn.auth_end_time,
|
||||
}
|
||||
|
||||
options = self.opts.copy()
|
||||
options['out_file'] = f.name
|
||||
options['skip_identical'] = True
|
||||
s = SwiftService()
|
||||
with mock.patch('swiftclient.service.time', side_effect=range(3)):
|
||||
with mock.patch('swiftclient.service.get_conn',
|
||||
@ -1239,11 +1359,7 @@ class TestServiceDownload(testtools.TestCase):
|
||||
conn=mock_conn,
|
||||
container='test_c',
|
||||
obj='test_o',
|
||||
options={'out_file': f.name,
|
||||
'header': {},
|
||||
'no_download': True,
|
||||
'yes_all': False,
|
||||
'skip_identical': True})
|
||||
options=options)
|
||||
|
||||
self._assertDictEqual(r, expected_r)
|
||||
|
||||
@ -1323,6 +1439,9 @@ class TestServiceDownload(testtools.TestCase):
|
||||
'auth_end_time': mock_conn.auth_end_time,
|
||||
}
|
||||
|
||||
options = self.opts.copy()
|
||||
options['out_file'] = f.name
|
||||
options['skip_identical'] = True
|
||||
s = SwiftService()
|
||||
with mock.patch('swiftclient.service.time', side_effect=range(3)):
|
||||
with mock.patch('swiftclient.service.get_conn',
|
||||
@ -1331,11 +1450,7 @@ class TestServiceDownload(testtools.TestCase):
|
||||
conn=mock_conn,
|
||||
container='test_c',
|
||||
obj='test_o',
|
||||
options={'out_file': f.name,
|
||||
'header': {},
|
||||
'no_download': True,
|
||||
'yes_all': False,
|
||||
'skip_identical': True})
|
||||
options=options)
|
||||
|
||||
self._assertDictEqual(r, expected_r)
|
||||
self.assertEqual(mock_conn.get_object.mock_calls, [
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user