openstack/python-swiftclient
Code Issues Proposed changes

Merge "Fix crash when downloading a pseudo-directory"

Browse Source
This commit is contained in:
Jenkins
2014-08-17 19:08:15 +00:00
committed by Gerrit Code Review
parent 23474b7d11 3d56b65c89
commit eedb0d4ab5
2 changed files with 30 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
swiftclient/shell.py
View File

@@ -24,7 +24,7 @@ from errno import EEXIST, ENOENT
from hashlib import md5 from hashlib import md5
from optparse import OptionParser, OptionGroup, SUPPRESS_HELP from optparse import OptionParser, OptionGroup, SUPPRESS_HELP
from os import environ, listdir, makedirs, utime, _exit as os_exit from os import environ, listdir, makedirs, utime, _exit as os_exit
from os.path import dirname, getmtime, getsize, isdir, join, \ from os.path import basename, dirname, getmtime, getsize, isdir, join, \
sep as os_path_sep sep as os_path_sep
from random import shuffle from random import shuffle
from sys import argv as sys_argv, exit, stderr, stdout from sys import argv as sys_argv, exit, stderr, stdout
@@ -381,7 +381,9 @@ def st_download(parser, args, thread_manager):
content_length = None content_length = None
etag = headers.get('etag') etag = headers.get('etag')
md5sum = None md5sum = None
make_dir = not options.no_download and out_file != "-" pseudodir = False
no_file = options.no_download
make_dir = not no_file and out_file != "-"
if content_type.split(';', 1)[0] == 'text/directory': if content_type.split(';', 1)[0] == 'text/directory':
if make_dir and not isdir(path): if make_dir and not isdir(path):
mkdirs(path) mkdirs(path)
@@ -397,24 +399,28 @@ def st_download(parser, args, thread_manager):
dirpath = dirname(path) dirpath = dirname(path)
if make_dir and dirpath and not isdir(dirpath): if make_dir and dirpath and not isdir(dirpath):
mkdirs(dirpath) mkdirs(dirpath)
if not options.no_download: if not no_file:
if out_file == "-": if out_file == "-":
fp = stdout fp = stdout
elif out_file: elif out_file:
fp = open(out_file, 'wb') fp = open(out_file, 'wb')
else: else:
fp = open(path, 'wb') if basename(path):
fp = open(path, 'wb')
else:
pseudodir = True
no_file = True
read_length = 0 read_length = 0
if 'x-object-manifest' not in headers and \ if 'x-object-manifest' not in headers and \
'x-static-large-object' not in headers: 'x-static-large-object' not in headers:
md5sum = md5() md5sum = md5()
for chunk in body: for chunk in body:
if not options.no_download: if not no_file:
fp.write(chunk) fp.write(chunk)
read_length += len(chunk) read_length += len(chunk)
if md5sum: if md5sum:
md5sum.update(chunk) md5sum.update(chunk)
if not options.no_download: if not no_file:
fp.close() fp.close()
if md5sum and md5sum.hexdigest() != etag: if md5sum and md5sum.hexdigest() != etag:
thread_manager.error('%s: md5sum != etag, %s != %s', thread_manager.error('%s: md5sum != etag, %s != %s',
@@ -424,8 +430,7 @@ def st_download(parser, args, thread_manager):
'%s: read_length != content_length, %d != %d', '%s: read_length != content_length, %d != %d',
path, read_length, content_length) path, read_length, content_length)
if 'x-object-meta-mtime' in headers and not options.out_file \ if 'x-object-meta-mtime' in headers and not options.out_file \
and not options.no_download: and not no_file:
mtime = float(headers['x-object-meta-mtime']) mtime = float(headers['x-object-meta-mtime'])
utime(path, (mtime, mtime)) utime(path, (mtime, mtime))
if options.verbose: if options.verbose:
@@ -434,10 +439,15 @@ def st_download(parser, args, thread_manager):
headers_receipt = headers_receipt - start_time headers_receipt = headers_receipt - start_time
total_time = finish_time - start_time total_time = finish_time - start_time
download_time = total_time - auth_time download_time = total_time - auth_time
time_str = ('auth %.3fs, headers %.3fs, total %.3fs, ' if pseudodir:
'%.3f MB/s' % ( time_str = (
auth_time, headers_receipt, total_time, 'auth %.3fs, headers %.3fs, total %.3fs, pseudo' % (
float(read_length) / download_time / 1000000)) auth_time, headers_receipt, total_time))
else:
time_str = (
'auth %.3fs, headers %.3fs, total %.3fs, %.3f MB/s' % (
auth_time, headers_receipt, total_time,
float(read_length) / download_time / 1000000))
if conn.attempts > 1: if conn.attempts > 1:
thread_manager.print_msg('%s [%s after %d attempts]', path, thread_manager.print_msg('%s [%s after %d attempts]', path,
time_str, conn.attempts) time_str, conn.attempts)

11
tests/unit/test_shell.py
View File

@@ -183,6 +183,7 @@ class TestShell(unittest.TestCase):
# Test downloading whole container # Test downloading whole container
connection.return_value.get_container.side_effect = [ connection.return_value.get_container.side_effect = [
[None, [{'name': 'object'}]], [None, [{'name': 'object'}]],
[None, [{'name': 'pseudo/'}]],
[None, []], [None, []],
] ]
connection.return_value.auth_end_time = 0 connection.return_value.auth_end_time = 0
@@ -191,9 +192,13 @@ class TestShell(unittest.TestCase):
with mock.patch(BUILTIN_OPEN) as mock_open: with mock.patch(BUILTIN_OPEN) as mock_open:
argv = ["", "download", "container"] argv = ["", "download", "container"]
swiftclient.shell.main(argv) swiftclient.shell.main(argv)
connection.return_value.get_object.assert_called_with( calls = [mock.call('container', 'object',
'container', 'object', headers={}, resp_chunk_size=65536) headers={}, resp_chunk_size=65536),
mock_open.assert_called_with('object', 'wb') mock.call('container', 'pseudo/',
headers={}, resp_chunk_size=65536)]
connection.return_value.get_object.assert_has_calls(
calls, any_order=True)
mock_open.assert_called_once_with('object', 'wb')
# Test downloading single object # Test downloading single object
with mock.patch(BUILTIN_OPEN) as mock_open: with mock.patch(BUILTIN_OPEN) as mock_open: