openstack/python-swiftclient
Code Issues Proposed changes

Fix download bandwidth for swift command.

Browse Source 
We noticed a difference in download bandwidth while using
authentication details and pre-auth information for download.
Download using authentication details included authentication
time into the download bandwidth at each call of the function.
This time should have been excluded from download bandwidth.

This patch set is adding a timer variable to the client library.
That allows to accurately measure and store the time of each action.
Then while printing the download bandwith in the swift command
authentication time is excluded from bandwidth.

Change-Id: I63df9023e169f637f120ae9e25dac9c90a4e75a0
Fixes: bug #1197443
This commit is contained in:
jola-mirecka 2013-09-04 19:32:09 +01:00 committed by Darrell Bishop
parent a253b5cecd
commit 9b7e24c449
2 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
bin/swift
View File

@ -331,7 +331,7 @@ def st_download(parser, args, thread_manager):
headers, body = \ headers, body = \
conn.get_object(container, obj, resp_chunk_size=65536, conn.get_object(container, obj, resp_chunk_size=65536,
headers=req_headers) headers=req_headers)
header_receipt = time() headers_receipt = time()
content_type = headers.get('content-type') content_type = headers.get('content-type')
if 'content-length' in headers: if 'content-length' in headers:
content_length = int(headers.get('content-length')) content_length = int(headers.get('content-length'))
@ -391,9 +391,14 @@ def st_download(parser, args, thread_manager):
utime(path, (mtime, mtime)) utime(path, (mtime, mtime))
if options.verbose: if options.verbose:
finish_time = time() finish_time = time()
time_str = 'headers %.3fs, total %.3fs, %.3f MB/s' % ( auth_time = conn.auth_end_time - start_time
header_receipt - start_time, finish_time - start_time, headers_receipt = headers_receipt - start_time
float(read_length) / (finish_time - start_time) / 1000000) total_time = finish_time - start_time
download_time = total_time - auth_time
time_str = ('auth %.3fs, headers %.3fs, total %.3fs, '
'%.3f MB/s' % (
auth_time, headers_receipt, total_time,
float(read_length) / download_time / 1000000))
if conn.attempts > 1: if conn.attempts > 1:
thread_manager.print_msg('%s [%s after %d attempts]', path, thread_manager.print_msg('%s [%s after %d attempts]', path,
time_str, conn.attempts) time_str, conn.attempts)
@ -460,9 +465,10 @@ def st_download(parser, args, thread_manager):
thread_manager.error('Account not found') thread_manager.error('Account not found')
elif len(args) == 1: elif len(args) == 1:
if '/' in args[0]: if '/' in args[0]:
print >> stderr, ('WARNING: / in container name; you ' print >> stderr, (
'might have meant %r instead of %r.' % ( 'WARNING: / in container name; you might have meant '
args[0].replace('/', ' ', 1), args[0])) '%r instead of %r.' % (
args[0].replace('/', ' ', 1), args[0]))
container_queue.put((args[0], object_queue, options.prefix)) container_queue.put((args[0], object_queue, options.prefix))
else: else:
if len(args) == 2: if len(args) == 2:

4
swiftclient/client.py
View File

@ -26,7 +26,7 @@ from functools import wraps
from urllib import quote as _quote from urllib import quote as _quote
from urlparse import urlparse, urlunparse from urlparse import urlparse, urlunparse
from httplib import HTTPException, HTTPConnection, HTTPSConnection from httplib import HTTPException, HTTPConnection, HTTPSConnection
from time import sleep from time import sleep, time
from swiftclient.exceptions import ClientException, InvalidHeadersException from swiftclient.exceptions import ClientException, InvalidHeadersException
@ -1071,6 +1071,7 @@ class Connection(object):
self.cacert = cacert self.cacert = cacert
self.insecure = insecure self.insecure = insecure
self.ssl_compression = ssl_compression self.ssl_compression = ssl_compression
self.auth_end_time = 0
def get_auth(self): def get_auth(self):
return get_auth(self.authurl, self.user, self.key, return get_auth(self.authurl, self.user, self.key,
@ -1104,6 +1105,7 @@ class Connection(object):
if not self.url or not self.token: if not self.url or not self.token:
self.url, self.token = self.get_auth() self.url, self.token = self.get_auth()
self.http_conn = None self.http_conn = None
self.auth_end_time = time()
if not self.http_conn: if not self.http_conn:
self.http_conn = self.http_connection() self.http_conn = self.http_connection()
kwargs['http_conn'] = self.http_conn kwargs['http_conn'] = self.http_conn