Added option to disable SQLite db preallocation

Added option to disable SQLite db preallocation. This can be very
useful on pure ssd account/container servers where the extra space is
worth more than the lesser fragmentation.

Change-Id: I8fbb028a9b6143775b25b343e97896497a8b63a9

doc/source/deployment_guide.rst

@@ -425,6 +425,10 @@ bind_ip             0.0.0.0     IP Address for server to bind to
 bind_port           6002        Port for server to bind to
 workers             1           Number of workers to fork
 user                swift       User to run as
+db_preallocation    on          Normally Swift will try to preallocate disk
+                                space for new SQLite databases to decrease
+                                fragmentation (at the cost of disk usage). You
+                                may turn this feature off here.
 ==================  ==========  =============================================
 
 [account-server]

etc/account-server.conf-sample

@@ -11,6 +11,10 @@
 # log_name = swift
 # log_facility = LOG_LOCAL0
 # log_level = INFO
+# Normally Swift will try to preallocate disk space for new SQLite databases to
+# decrease fragmentation (at the cost of disk usage). You may turn this feature
+# off here.
+# db_preallocation = on
 
 [pipeline:main]
 pipeline = account-server

etc/container-server.conf-sample

@@ -14,6 +14,10 @@
 # log_name = swift
 # log_facility = LOG_LOCAL0
 # log_level = INFO
+# Normally Swift will try to preallocate disk space for new SQLite databases to
+# decrease fragmentation (at the cost of disk usage). You may turn this feature
+# off here.
+# db_preallocation = on
 
 [pipeline:main]
 pipeline = container-server

swift/account/auditor.py

@@ -17,9 +17,11 @@ import os
 import time
 from random import random
 
+import swift.common.db
 from swift.account import server as account_server
 from swift.common.db import AccountBroker
-from swift.common.utils import get_logger, audit_location_generator
+from swift.common.utils import get_logger, audit_location_generator, \
+    TRUE_VALUES
 from swift.common.daemon import Daemon
 
 from eventlet import Timeout
@@ -37,6 +39,8 @@ class AccountAuditor(Daemon):
         self.interval = int(conf.get('interval', 1800))
         self.account_passes = 0
         self.account_failures = 0
+        swift.common.db.DB_PREALLOCATION = \
+            conf.get('db_preallocation', 't').lower() in TRUE_VALUES
 
     def run_forever(self, *args, **kwargs):
         """Run the account audit until stopped."""

swift/account/reaper.py

@@ -21,12 +21,13 @@ from time import time
 
 from eventlet import GreenPool, sleep, Timeout
 
+import swift.common.db
 from swift.account.server import DATADIR
 from swift.common.db import AccountBroker
 from swift.common.direct_client import ClientException, \
     direct_delete_container, direct_delete_object, direct_get_container
 from swift.common.ring import Ring
-from swift.common.utils import get_logger, whataremyips
+from swift.common.utils import get_logger, whataremyips, TRUE_VALUES
 from swift.common.daemon import Daemon
 
 
@@ -69,6 +70,8 @@ class AccountReaper(Daemon):
         self.container_concurrency = self.object_concurrency = \
             sqrt(self.concurrency)
         self.container_pool = GreenPool(size=self.container_concurrency)
+        swift.common.db.DB_PREALLOCATION = \
+            conf.get('db_preallocation', 't').lower() in TRUE_VALUES
         self.delay_reaping = int(conf.get('delay_reaping') or 0)
 
     def get_account_ring(self):

swift/account/server.py

@@ -29,9 +29,10 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, \
     HTTPPreconditionFailed, HTTPConflict
 import simplejson
 
+import swift.common.db
 from swift.common.db import AccountBroker
 from swift.common.utils import get_logger, get_param, hash_path, \
-    normalize_timestamp, split_path, storage_directory
+    normalize_timestamp, split_path, storage_directory, TRUE_VALUES
 from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \
     check_mount, check_float, check_utf8
 from swift.common.db_replicator import ReplicatorRpc
@@ -52,6 +53,8 @@ class AccountController(object):
             self.mount_check, logger=self.logger)
         self.auto_create_account_prefix = \
             conf.get('auto_create_account_prefix') or '.'
+        swift.common.db.DB_PREALLOCATION = \
+            conf.get('db_preallocation', 't').lower() in TRUE_VALUES
 
     def _get_account_broker(self, drive, part, account):
         hsh = hash_path(account)

swift/common/db.py

@@ -37,6 +37,8 @@ from swift.common.utils import normalize_timestamp, renamer, \
 from swift.common.exceptions import LockTimeout
 
 
+#: Whether calls will be made to preallocate disk space for database files.
+DB_PREALLOCATION = True
 #: Timeout for trying to connect to a DB
 BROKER_TIMEOUT = 25
 #: Pickle protocol to use
@@ -508,7 +510,7 @@ class DatabaseBroker(object):
         within 512k of a boundary, it allocates to the next boundary.
         Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after.
         """
-        if self.db_file == ':memory:':
+        if not DB_PREALLOCATION or self.db_file == ':memory:':
             return
         MB = (1024 * 1024)
 

swift/container/auditor.py

@@ -19,9 +19,11 @@ from random import random
 
 from eventlet import Timeout
 
+import swift.common.db
 from swift.container import server as container_server
 from swift.common.db import ContainerBroker
-from swift.common.utils import get_logger, audit_location_generator
+from swift.common.utils import get_logger, audit_location_generator, \
+    TRUE_VALUES
 from swift.common.daemon import Daemon
 
 
@@ -38,6 +40,8 @@ class ContainerAuditor(Daemon):
         swift_dir = conf.get('swift_dir', '/etc/swift')
         self.container_passes = 0
         self.container_failures = 0
+        swift.common.db.DB_PREALLOCATION = \
+            conf.get('db_preallocation', 't').lower() in TRUE_VALUES
 
     def run_forever(self, *args, **kwargs):
         """Run the container audit until stopped."""

swift/container/server.py

@@ -29,6 +29,7 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \
     HTTPCreated, HTTPInternalServerError, HTTPNoContent, \
     HTTPNotFound, HTTPPreconditionFailed, HTTPMethodNotAllowed
 
+import swift.common.db
 from swift.common.db import ContainerBroker
 from swift.common.utils import get_logger, get_param, hash_path, \
     normalize_timestamp, storage_directory, split_path, validate_sync_to, \
@@ -65,6 +66,8 @@ class ContainerController(object):
             conf.get('auto_create_account_prefix') or '.'
         if conf.get('allow_versions', 'f').lower() in TRUE_VALUES:
             self.save_headers.append('x-versions-location')
+        swift.common.db.DB_PREALLOCATION = \
+            conf.get('db_preallocation', 't').lower() in TRUE_VALUES
 
     def _get_container_broker(self, drive, part, account, container):
         """

swift/container/sync.py

@@ -20,6 +20,7 @@ from struct import unpack_from
 
 from eventlet import sleep, Timeout
 
+import swift.common.db
 from swift.container import server as container_server
 from swift.common.client import ClientException, delete_object, put_object, \
     quote
@@ -178,6 +179,8 @@ class ContainerSync(Daemon):
         self.object_ring = object_ring or Ring(swift_dir, ring_name='object')
         self._myips = whataremyips()
         self._myport = int(conf.get('bind_port', 6001))
+        swift.common.db.DB_PREALLOCATION = \
+            conf.get('db_preallocation', 't').lower() in TRUE_VALUES
 
     def run_forever(self):
         """

swift/container/updater.py

@@ -23,12 +23,13 @@ from tempfile import mkstemp
 
 from eventlet import spawn, patcher, Timeout
 
+import swift.common.db
 from swift.container.server import DATADIR
 from swift.common.bufferedhttp import http_connect
 from swift.common.db import ContainerBroker
 from swift.common.exceptions import ConnectionTimeout
 from swift.common.ring import Ring
-from swift.common.utils import get_logger, whataremyips
+from swift.common.utils import get_logger, whataremyips, TRUE_VALUES
 from swift.common.daemon import Daemon
 
 
@@ -55,6 +56,8 @@ class ContainerUpdater(Daemon):
         self.account_suppression_time = \
             float(conf.get('account_suppression_time', 60))
         self.new_account_suppressions = None
+        swift.common.db.DB_PREALLOCATION = \
+            conf.get('db_preallocation', 't').lower() in TRUE_VALUES
 
     def get_account_ring(self):
         """Get the account ring.  Load it if it hasn't been yet."""

test/unit/common/test_db.py

@@ -97,6 +97,13 @@ class TestDatabaseBroker(unittest.TestCase):
         rmtree(self.testdir, ignore_errors=1)
         os.mkdir(self.testdir)
 
+    def test_DB_PREALLOCATION_setting(self):
+        u = uuid4().hex
+        b = DatabaseBroker(u)
+        self.assertRaises(OSError, b._preallocate)
+        swift.common.db.DB_PREALLOCATION = False
+        b._preallocate()
+
     def tearDown(self):
         rmtree(self.testdir, ignore_errors=1)