openstack/python-swiftclient
Code Issues Proposed changes

Buffer reads from disk

Browse Source 
Otherwise, Python defaults to 8k reads which seems kinda terrible.

Change-Id: I3160626e947083af487fd1c3cb0aa6a62646527b
Closes-Bug: #1671621
This commit is contained in:
Tim Burke 2017-03-15 18:05:38 +00:00
parent cde73c196d
commit 638d7c789c
3 changed files with 20 additions and 19 deletions

19
swiftclient/service.py

@ -50,6 +50,7 @@ from swiftclient.exceptions import ClientException
from swiftclient.multithreading import MultiThreadingManager
DISK_BUFFER = 2 ** 16
logger = logging.getLogger("swiftclient.service")
@ -1126,14 +1127,14 @@ class SwiftService(object):
if options['skip_identical']:
filename = out_file if out_file else path
try:
fp = open(filename, 'rb')
fp = open(filename, 'rb', DISK_BUFFER)
except IOError:
pass
else:
with fp:
md5sum = md5()
while True:
data = fp.read(65536)
data = fp.read(DISK_BUFFER)
if not data:
break
md5sum.update(data)
@ -1141,7 +1142,7 @@ class SwiftService(object):
try:
start_time = time()
get_args = {'resp_chunk_size': 65536,
get_args = {'resp_chunk_size': DISK_BUFFER,
'headers': req_headers,
'response_dict': results_dict}
if options['skip_identical']:
@ -1224,10 +1225,10 @@ class SwiftService(object):
if not no_file:
if out_file:
fp = open(out_file, 'wb')
fp = open(out_file, 'wb', DISK_BUFFER)
else:
if basename(path):
fp = open(path, 'wb')
fp = open(path, 'wb', DISK_BUFFER)
else:
pseudodir = True
@ -1733,7 +1734,7 @@ class SwiftService(object):
}
fp = None
try:
fp = open(path, 'rb')
fp = open(path, 'rb', DISK_BUFFER)
fp.seek(segment_start)
contents = LengthWrapper(fp, segment_size, md5=options['checksum'])
@ -1811,7 +1812,7 @@ class SwiftService(object):
def _is_identical(self, chunk_data, path):
try:
fp = open(path, 'rb')
fp = open(path, 'rb', DISK_BUFFER)
except IOError:
return False
@ -1820,7 +1821,7 @@ class SwiftService(object):
to_read = chunk['bytes']
md5sum = md5()
while to_read:
data = fp.read(min(65536, to_read))
data = fp.read(min(DISK_BUFFER, to_read))
if not data:
return False
md5sum.update(data)
@ -2032,7 +2033,7 @@ class SwiftService(object):
try:
if path is not None:
content_length = getsize(path)
fp = open(path, 'rb')
fp = open(path, 'rb', DISK_BUFFER)
contents = LengthWrapper(fp,
content_length,
md5=options['checksum'])

8
tests/unit/test_service.py

@ -1951,7 +1951,7 @@ class TestServiceDownload(_TestServiceBase):
'headers_receipt': 3
}
)
mock_open.assert_called_once_with('test_o', 'wb')
mock_open.assert_called_once_with('test_o', 'wb', 65536)
written_content.write.assert_called_once_with(b'objcontent')
mock_conn.get_object.assert_called_once_with(
@ -1995,7 +1995,7 @@ class TestServiceDownload(_TestServiceBase):
'headers_receipt': 3
}
)
mock_open.assert_called_once_with('test_o', 'wb')
mock_open.assert_called_once_with('test_o', 'wb', 65536)
mock_utime.assert_called_once_with(
'test_o', (1454113727.682512, 1454113727.682512))
written_content.write.assert_called_once_with(b'objcontent')
@ -2041,7 +2041,7 @@ class TestServiceDownload(_TestServiceBase):
'headers_receipt': 3
}
)
mock_open.assert_called_once_with('test_o', 'wb')
mock_open.assert_called_once_with('test_o', 'wb', 65536)
self.assertEqual(0, len(mock_utime.mock_calls))
written_content.write.assert_called_once_with(b'objcontent')
@ -2087,7 +2087,7 @@ class TestServiceDownload(_TestServiceBase):
'headers_receipt': 3
}
)
mock_open.assert_called_once_with('test_o', 'wb')
mock_open.assert_called_once_with('test_o', 'wb', 65536)
self.assertEqual([], mock_utime.mock_calls)
written_content.write.assert_called_once_with(b'objcontent')

12
tests/unit/test_shell.py

@ -473,7 +473,7 @@ class TestShell(unittest.TestCase):
response_dict={})]
connection.return_value.get_object.assert_has_calls(
calls, any_order=True)
mock_open.assert_called_once_with('object', 'wb')
mock_open.assert_called_once_with('object', 'wb', 65536)
self.assertEqual([mock.call('pseudo')], makedirs.mock_calls)
makedirs.reset_mock()
@ -490,7 +490,7 @@ class TestShell(unittest.TestCase):
connection.return_value.get_object.assert_called_with(
'container', 'object', headers={}, resp_chunk_size=65536,
response_dict={})
mock_open.assert_called_with('object', 'wb')
mock_open.assert_called_with('object', 'wb', 65536)
self.assertEqual([], makedirs.mock_calls)
# Test downloading without md5 checks
@ -507,7 +507,7 @@ class TestShell(unittest.TestCase):
connection.return_value.get_object.assert_called_with(
'container', 'object', headers={}, resp_chunk_size=65536,
response_dict={})
mock_open.assert_called_with('object', 'wb')
mock_open.assert_called_with('object', 'wb', 65536)
sr.assert_called_once_with('object', mock.ANY, mock.ANY, False)
self.assertEqual([], makedirs.mock_calls)
@ -553,7 +553,7 @@ class TestShell(unittest.TestCase):
mock_shuffle.assert_any_call(['container'])
mock_shuffle.assert_any_call(['object'])
mock_shuffle.assert_any_call(['pseudo/'])
mock_open.assert_called_once_with('container/object', 'wb')
mock_open.assert_called_once_with('container/object', 'wb', 65536)
self.assertEqual([
mock.call('container'),
mock.call('container/pseudo'),
@ -577,7 +577,7 @@ class TestShell(unittest.TestCase):
argv = ["", "download", "--all", "--no-shuffle"]
swiftclient.shell.main(argv)
self.assertEqual(0, mock_shuffle.call_count)
mock_open.assert_called_once_with('container/object', 'wb')
mock_open.assert_called_once_with('container/object', 'wb', 65536)
self.assertEqual([
mock.call('container'),
mock.call('container/pseudo'),
@ -610,7 +610,7 @@ class TestShell(unittest.TestCase):
response_dict={})]
connection.return_value.get_object.assert_has_calls(
calls, any_order=True)
mock_open.assert_called_once_with('object', 'wb')
mock_open.assert_called_once_with('object', 'wb', 65536)
self.assertEqual([
mock.call('pseudo'),
], mock_mkdir.mock_calls)