Add ability to download without writing to disk.

This patch adds a "--no-download" option to the "download" command.
When given, all writing to disk is bypassed, while still actually
downloading the data and validating etags.

This can be handy when you're using the swift command-line client to
test out a swift cluster and don't want client-side disk writing to be a
bottle-neck (but you still want to know about any etag validation
failures).

Change-Id: I0a511f473a64820161d1eb529b995900742794f2
This commit is contained in:
Darrell Bishop 2012-08-17 10:51:25 -07:00
parent 3207e36984
commit c94ca0e298

@ -329,6 +329,9 @@ def st_download(parser, args, print_queue, error_queue):
parser.add_option('', '--container-threads', type=int,
default=10, help='Number of threads to use for '
'listing containers')
parser.add_option('', '--no-download', action='store_true',
default=False, help="Perform download(s), but don't "
"actually write anything to disk")
(options, args) = parse_args(parser, args)
args = args[1:]
if options.out_file == '-':
@ -365,7 +368,7 @@ def st_download(parser, args, print_queue, error_queue):
if path[:1] in ('/', '\\'):
path = path[1:]
md5sum = None
make_dir = out_file != "-"
make_dir = not options.no_download and out_file != "-"
if content_type.split(';', 1)[0] == 'text/directory':
if make_dir and not isdir(path):
mkdirs(path)
@ -380,28 +383,33 @@ def st_download(parser, args, print_queue, error_queue):
dirpath = dirname(path)
if make_dir and dirpath and not isdir(dirpath):
mkdirs(dirpath)
if out_file == "-":
fp = stdout
elif out_file:
fp = open(out_file, 'wb')
else:
fp = open(path, 'wb')
if not options.no_download:
if out_file == "-":
fp = stdout
elif out_file:
fp = open(out_file, 'wb')
else:
fp = open(path, 'wb')
read_length = 0
if 'x-object-manifest' not in headers:
md5sum = md5()
for chunk in body:
fp.write(chunk)
if not options.no_download:
fp.write(chunk)
read_length += len(chunk)
if md5sum:
md5sum.update(chunk)
fp.close()
if not options.no_download:
fp.close()
if md5sum and md5sum.hexdigest() != etag:
error_queue.put('%s: md5sum != etag, %s != %s' %
(path, md5sum.hexdigest(), etag))
if content_length is not None and read_length != content_length:
error_queue.put('%s: read_length != content_length, %d != %d' %
(path, read_length, content_length))
if 'x-object-meta-mtime' in headers and not options.out_file:
if 'x-object-meta-mtime' in headers and not options.out_file \
and not options.no_download:
mtime = float(headers['x-object-meta-mtime'])
utime(path, (mtime, mtime))
if options.verbose: