Buffer reads from disk

Otherwise, Python defaults to 8k reads which seems kinda terrible.

Change-Id: I3160626e947083af487fd1c3cb0aa6a62646527b
Closes-Bug: #1671621

swiftclient/service.py

@@ -50,6 +50,7 @@ from swiftclient.exceptions import ClientException
 from swiftclient.multithreading import MultiThreadingManager
 
 
+DISK_BUFFER = 2 ** 16
 logger = logging.getLogger("swiftclient.service")
 
 
@@ -1126,14 +1127,14 @@ class SwiftService(object):
         if options['skip_identical']:
             filename = out_file if out_file else path
             try:
-                fp = open(filename, 'rb')
+                fp = open(filename, 'rb', DISK_BUFFER)
             except IOError:
                 pass
             else:
                 with fp:
                     md5sum = md5()
                     while True:
-                        data = fp.read(65536)
+                        data = fp.read(DISK_BUFFER)
                         if not data:
                             break
                         md5sum.update(data)
@@ -1141,7 +1142,7 @@ class SwiftService(object):
 
         try:
             start_time = time()
-            get_args = {'resp_chunk_size': 65536,
+            get_args = {'resp_chunk_size': DISK_BUFFER,
                         'headers': req_headers,
                         'response_dict': results_dict}
             if options['skip_identical']:
@@ -1224,10 +1225,10 @@ class SwiftService(object):
 
                     if not no_file:
                         if out_file:
-                            fp = open(out_file, 'wb')
+                            fp = open(out_file, 'wb', DISK_BUFFER)
                         else:
                             if basename(path):
-                                fp = open(path, 'wb')
+                                fp = open(path, 'wb', DISK_BUFFER)
                             else:
                                 pseudodir = True
 
@@ -1733,7 +1734,7 @@ class SwiftService(object):
         }
         fp = None
         try:
-            fp = open(path, 'rb')
+            fp = open(path, 'rb', DISK_BUFFER)
             fp.seek(segment_start)
 
             contents = LengthWrapper(fp, segment_size, md5=options['checksum'])
@@ -1811,7 +1812,7 @@ class SwiftService(object):
 
     def _is_identical(self, chunk_data, path):
         try:
-            fp = open(path, 'rb')
+            fp = open(path, 'rb', DISK_BUFFER)
         except IOError:
             return False
 
@@ -1820,7 +1821,7 @@ class SwiftService(object):
                 to_read = chunk['bytes']
                 md5sum = md5()
                 while to_read:
-                    data = fp.read(min(65536, to_read))
+                    data = fp.read(min(DISK_BUFFER, to_read))
                     if not data:
                         return False
                     md5sum.update(data)
@@ -2032,7 +2033,7 @@ class SwiftService(object):
                 try:
                     if path is not None:
                         content_length = getsize(path)
-                        fp = open(path, 'rb')
+                        fp = open(path, 'rb', DISK_BUFFER)
                         contents = LengthWrapper(fp,
                                                  content_length,
                                                  md5=options['checksum'])

tests/unit/test_service.py

@@ -1951,7 +1951,7 @@ class TestServiceDownload(_TestServiceBase):
                     'headers_receipt': 3
                 }
             )
-            mock_open.assert_called_once_with('test_o', 'wb')
+            mock_open.assert_called_once_with('test_o', 'wb', 65536)
             written_content.write.assert_called_once_with(b'objcontent')
 
         mock_conn.get_object.assert_called_once_with(
@@ -1995,7 +1995,7 @@ class TestServiceDownload(_TestServiceBase):
                     'headers_receipt': 3
                 }
             )
-            mock_open.assert_called_once_with('test_o', 'wb')
+            mock_open.assert_called_once_with('test_o', 'wb', 65536)
             mock_utime.assert_called_once_with(
                 'test_o', (1454113727.682512, 1454113727.682512))
             written_content.write.assert_called_once_with(b'objcontent')
@@ -2041,7 +2041,7 @@ class TestServiceDownload(_TestServiceBase):
                     'headers_receipt': 3
                 }
             )
-            mock_open.assert_called_once_with('test_o', 'wb')
+            mock_open.assert_called_once_with('test_o', 'wb', 65536)
             self.assertEqual(0, len(mock_utime.mock_calls))
             written_content.write.assert_called_once_with(b'objcontent')
 
@@ -2087,7 +2087,7 @@ class TestServiceDownload(_TestServiceBase):
                     'headers_receipt': 3
                 }
             )
-            mock_open.assert_called_once_with('test_o', 'wb')
+            mock_open.assert_called_once_with('test_o', 'wb', 65536)
             self.assertEqual([], mock_utime.mock_calls)
             written_content.write.assert_called_once_with(b'objcontent')
 

tests/unit/test_shell.py

@@ -473,7 +473,7 @@ class TestShell(unittest.TestCase):
                            response_dict={})]
         connection.return_value.get_object.assert_has_calls(
             calls, any_order=True)
-        mock_open.assert_called_once_with('object', 'wb')
+        mock_open.assert_called_once_with('object', 'wb', 65536)
         self.assertEqual([mock.call('pseudo')], makedirs.mock_calls)
         makedirs.reset_mock()
 
@@ -490,7 +490,7 @@ class TestShell(unittest.TestCase):
         connection.return_value.get_object.assert_called_with(
             'container', 'object', headers={}, resp_chunk_size=65536,
             response_dict={})
-        mock_open.assert_called_with('object', 'wb')
+        mock_open.assert_called_with('object', 'wb', 65536)
         self.assertEqual([], makedirs.mock_calls)
 
         # Test downloading without md5 checks
@@ -507,7 +507,7 @@ class TestShell(unittest.TestCase):
         connection.return_value.get_object.assert_called_with(
             'container', 'object', headers={}, resp_chunk_size=65536,
             response_dict={})
-        mock_open.assert_called_with('object', 'wb')
+        mock_open.assert_called_with('object', 'wb', 65536)
         sr.assert_called_once_with('object', mock.ANY, mock.ANY, False)
         self.assertEqual([], makedirs.mock_calls)
 
@@ -553,7 +553,7 @@ class TestShell(unittest.TestCase):
         mock_shuffle.assert_any_call(['container'])
         mock_shuffle.assert_any_call(['object'])
         mock_shuffle.assert_any_call(['pseudo/'])
-        mock_open.assert_called_once_with('container/object', 'wb')
+        mock_open.assert_called_once_with('container/object', 'wb', 65536)
         self.assertEqual([
             mock.call('container'),
             mock.call('container/pseudo'),
@@ -577,7 +577,7 @@ class TestShell(unittest.TestCase):
                 argv = ["", "download", "--all", "--no-shuffle"]
                 swiftclient.shell.main(argv)
         self.assertEqual(0, mock_shuffle.call_count)
-        mock_open.assert_called_once_with('container/object', 'wb')
+        mock_open.assert_called_once_with('container/object', 'wb', 65536)
         self.assertEqual([
             mock.call('container'),
             mock.call('container/pseudo'),
@@ -610,7 +610,7 @@ class TestShell(unittest.TestCase):
                            response_dict={})]
         connection.return_value.get_object.assert_has_calls(
             calls, any_order=True)
-        mock_open.assert_called_once_with('object', 'wb')
+        mock_open.assert_called_once_with('object', 'wb', 65536)
         self.assertEqual([
             mock.call('pseudo'),
         ], mock_mkdir.mock_calls)