diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index c78ca2de5e..0e6b3cce5b 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -23,6 +23,8 @@ use = egg:swift#object # disk_chunk_size = 65536 # max_upload_time = 86400 # slow = 1 +# on PUTs, sync data every n MB +# mb_per_sync = 512 [object-replicator] # log_name = object-replicator diff --git a/swift/obj/server.py b/swift/obj/server.py index fe26eebf20..6a15aef35d 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -259,7 +259,7 @@ class ObjectController(object): self.log_requests = conf.get('log_requests', 't')[:1].lower() == 't' self.max_upload_time = int(conf.get('max_upload_time', 86400)) self.slow = int(conf.get('slow', 0)) - self.chunks_per_sync = int(conf.get('chunks_per_sync', 8000)) + self.bytes_per_sync = int(conf.get('mb_per_sync', 512)) * 1024 * 1024 def container_update(self, op, account, container, obj, headers_in, headers_out, objdevice): @@ -359,11 +359,10 @@ class ObjectController(object): upload_expiration = time.time() + self.max_upload_time etag = md5() upload_size = 0 + last_sync = 0 with file.mkstemp() as (fd, tmppath): if 'content-length' in request.headers: fallocate(fd, int(request.headers['content-length'])) - chunk_count = 0 - dropped_cache = 0 for chunk in iter(lambda: request.body_file.read( self.network_chunk_size), ''): upload_size += len(chunk) @@ -373,13 +372,11 @@ class ObjectController(object): while chunk: written = os.write(fd, chunk) chunk = chunk[written:] - chunk_count += 1 # For large files sync every 512MB (by default) written - if chunk_count % self.chunks_per_sync == 0: + if upload_size - last_sync >= self.bytes_per_sync: os.fdatasync(fd) - drop_buffer_cache(fd, dropped_cache, - upload_size - dropped_cache) - dropped_cache = upload_size + drop_buffer_cache(fd, last_sync, upload_size - last_sync) + last_sync = upload_size if 'content-length' in request.headers and \ int(request.headers['content-length']) != upload_size: diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 048b409c63..94a3b28266 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -56,7 +56,7 @@ class TestObjectController(unittest.TestCase): mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) conf = {'devices': self.testdir, 'mount_check': 'false'} self.object_controller = object_server.ObjectController(conf) - self.object_controller.chunks_per_sync = 1 + self.object_controller.bytes_per_sync = 1 def tearDown(self): """ Tear down for testing swift.object_server.ObjectController """