Add Barbican configs for SQLAlchemy pool settings

In order to support tuning of database operation in production, this
commit exposes pool-related SQLAlchemy parameters. This includes
enabling DEBUG/INFO logging from the pool (when it connects, how many
connections are active), the number of connection instances to keep,
the maximum number of connections over that to allow, and the
SQLAlchemy pool class to use.

Change-Id: I573147d16f61966eb74479b66077efa23c638a43

barbican/common/utils.py

@@ -17,6 +17,7 @@
 Common utilities for Barbican.
 """
 import collections
+import importlib
 import mimetypes
 import uuid
 
@@ -134,5 +135,16 @@ def generate_fullname_for(instance):
     return "{module}.{class_name}".format(module=module, class_name=class_name)
 
 
+def get_class_for(module_name, class_name):
+    """Create a Python class from its text-specified components."""
+    # Load the module via name, raising ImportError if module cannot be
+    # loaded.
+    python_module = importlib.import_module(module_name)
+
+    # Load and return the resolved Python class, raising AttributeError if
+    # class cannot be found.
+    return getattr(python_module, class_name)
+
+
 def generate_uuid():
     return str(uuid.uuid4())

barbican/model/repositories.py

@@ -76,6 +76,10 @@ db_opts = [
     cfg.StrOpt('sql_connection'),
     cfg.IntOpt('max_limit_paging', default=100),
     cfg.IntOpt('default_limit_paging', default=10),
+    cfg.StrOpt('sql_pool_class', default=None),
+    cfg.StrOpt('sql_pool_logging', default=False),
+    cfg.IntOpt('sql_pool_size', default=None),
+    cfg.IntOpt('sql_pool_max_overflow', default=None),
 ]
 
 CONF = cfg.CONF
@@ -102,9 +106,12 @@ def setup_database_engine_and_factory():
 
     LOG.info('Setting up database engine and session factory')
     LOG.debug('Sql connection = %s', CONF.sql_connection)
-    sa_logger = logging.getLogger('sqlalchemy.engine')
     if CONF.debug:
+        sa_logger = logging.getLogger('sqlalchemy.engine')
         sa_logger.setLevel(logging.DEBUG)
+    if CONF.sql_pool_logging:
+        pool_logger = logging.getLogger('sqlalchemy.pool')
+        pool_logger.setLevel(logging.DEBUG)
 
     _ENGINE = _get_engine(_ENGINE)
 
@@ -176,6 +183,13 @@ def _get_engine(engine):
             'pool_recycle': CONF.sql_idle_timeout,
             'echo': False,
             'convert_unicode': True}
+        if CONF.sql_pool_class:
+            engine_args['poolclass'] = utils.get_class_for(
+                'sqlalchemy.pool', CONF.sql_pool_class)
+        if CONF.sql_pool_size:
+            engine_args['pool_size'] = CONF.sql_pool_size
+        if CONF.sql_pool_max_overflow:
+            engine_args['max_overflow'] = CONF.sql_pool_max_overflow
 
         try:
             engine = _create_engine(connection, **engine_args)

barbican/tests/database_utils.py

@@ -27,7 +27,7 @@ def setup_in_memory_db():
     # Ensure we are using in-memory SQLite database, and creating tables.
     repositories.CONF.set_override("sql_connection", "sqlite:///:memory:")
     repositories.CONF.set_override("db_auto_create", True)
-    repositories.CONF.set_override("debug", False)
+    repositories.CONF.set_override("debug", True)
 
     # Ensure the connection is completely closed, so any previous in-memory
     # database can be removed prior to starting the next test run.

barbican/tests/model/repositories/test_repositories.py

@@ -240,7 +240,7 @@ class WhenTestingGetEnginePrivate(utils.BaseTestCase):
             exception_result.message)
 
     @mock.patch('barbican.model.repositories._create_engine')
-    def test_should_complete_with_no_alembic_create(
+    def test_should_complete_with_no_alembic_create_default_configs(
             self, mock_create_engine):
 
         repositories.CONF.set_override("db_auto_create", False)
@@ -251,6 +251,39 @@ class WhenTestingGetEnginePrivate(utils.BaseTestCase):
         repositories._get_engine(None)
 
         engine.connect.assert_called_once_with()
+        mock_create_engine.assert_called_once_with(
+            'connection',
+            pool_recycle=3600,
+            convert_unicode=True,
+            echo=False
+        )
+
+    @mock.patch('barbican.model.repositories._create_engine')
+    def test_should_complete_with_no_alembic_create_pool_configs(
+            self, mock_create_engine):
+
+        repositories.CONF.set_override("db_auto_create", False)
+        repositories.CONF.set_override(
+            "sql_pool_class", "QueuePool")
+        repositories.CONF.set_override("sql_pool_size", 22)
+        repositories.CONF.set_override("sql_pool_max_overflow", 11)
+
+        engine = mock.MagicMock()
+        mock_create_engine.return_value = engine
+
+        # Invoke method under test.
+        repositories._get_engine(None)
+
+        engine.connect.assert_called_once_with()
+        mock_create_engine.assert_called_once_with(
+            'connection',
+            pool_recycle=3600,
+            convert_unicode=True,
+            echo=False,
+            poolclass=sqlalchemy.pool.QueuePool,
+            pool_size=22,
+            max_overflow=11
+        )
 
 
 class WhenTestingAutoGenerateTables(utils.BaseTestCase):

etc/barbican/barbican-api.conf

@@ -49,6 +49,33 @@ sql_connection = sqlite:////var/lib/barbican/barbican.sqlite
 # before MySQL can drop the connection.
 sql_idle_timeout = 3600
 
+# Accepts a class imported from the sqlalchemy.pool module, and handles the
+# details of building the pool for you. If commented out, SQLAlchemy
+# will select based on the database dialect. Other options are QueuePool
+# (for SQLAlchemy-managed connections) and NullPool (to disabled SQLAlchemy
+# management of connections).
+# See http://docs.sqlalchemy.org/en/latest/core/pooling.html for more details.
+#sql_pool_class = QueuePool
+
+# Show SQLAlchemy pool-related debugging output in logs (sets DEBUG log level
+# output) if specified.
+#sql_pool_logging = True
+
+# Size of pool used by SQLAlchemy. This is the largest number of connections
+# that will be kept persistently in the pool. Can be set to 0 to indicate no
+# size limit. To disable pooling, use a NullPool with sql_pool_class instead.
+# Comment out to allow SQLAlchemy to select the default.
+#sql_pool_size = 5
+
+# The maximum overflow size of the pool used by SQLAlchemy. When the number of
+# checked-out connections reaches the size set in sql_pool_size, additional
+# connections will be returned up to this limit. It follows then that the
+# total number of simultaneous connections the pool will allow is
+# sql_pool_size + sql_pool_max_overflow. Can be set to -1 to indicate no
+# overflow limit, so no limit will be placed on the total number of concurrent
+# connections. Comment out to allow SQLAlchemy to select the default.
+#sql_pool_max_overflow = 10
+
 # Default page size for the 'limit' paging URL parameter.
 default_limit_paging = 10