Merge "Make Gnocchi connection pool configurable"

cloudkitty/collector/gnocchi.py

@@ -14,6 +14,7 @@
 #    under the License.
 #
 from datetime import timedelta
+import requests
 import six
 
 from gnocchiclient import auth as gauth
@@ -30,6 +31,7 @@ from voluptuous import Required
 from voluptuous import Schema
 
 from cloudkitty import collector
+from cloudkitty.common import custom_session
 from cloudkitty import dataframe
 from cloudkitty import utils as ck_utils
 from cloudkitty.utils import tz as tzutils
@@ -67,6 +69,12 @@ collector_gnocchi_opts = [
         default='RegionOne',
         help='Region Name',
     ),
+    cfg.IntOpt(
+        'http_pool_maxsize',
+        default=requests.adapters.DEFAULT_POOLSIZE,
+        help='If the value is not defined, we use the value defined by '
+             'requests.adapters.DEFAULT_POOLSIZE',
+    )
 ]
 
 ks_loading.register_session_conf_options(cfg.CONF, COLLECTOR_GNOCCHI_OPTS)
@@ -150,7 +158,9 @@ class GnocchiCollector(collector.BaseCollector):
 
         self._conn = gclient.Client(
             '1',
-            session_options={'auth': auth_plugin, 'verify': verify},
+            session=custom_session.create_custom_session(
+                {'auth': auth_plugin, 'verify': verify},
+                CONF.collector_gnocchi.http_pool_maxsize),
             adapter_options=adapter_options,
         )
 

cloudkitty/common/custom_session.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import logging
+import requests
+
+from keystoneauth1 import session as ks_session
+
+
+LOG = logging.getLogger(__name__)
+
+
+def create_custom_session(session_options, pool_size):
+    LOG.debug("Using custom connection pool size: %s", pool_size)
+    session = requests.Session()
+    session.adapters['http://'] = ks_session.TCPKeepAliveAdapter(
+        pool_maxsize=pool_size)
+    session.adapters['https://'] = ks_session.TCPKeepAliveAdapter(
+        pool_maxsize=pool_size)
+
+    return ks_session.Session(session=session, **session_options)

cloudkitty/fetcher/gnocchi.py

@@ -13,12 +13,15 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
+import requests
+
 from gnocchiclient import auth as gauth
 from gnocchiclient import client as gclient
 from keystoneauth1 import loading as ks_loading
 from oslo_config import cfg
 from oslo_log import log
 
+from cloudkitty.common import custom_session
 from cloudkitty import fetcher
 
 
@@ -61,6 +64,12 @@ gfetcher_opts = [
         default='RegionOne',
         help='Region Name',
     ),
+    cfg.IntOpt(
+        'http_pool_maxsize',
+        default=requests.adapters.DEFAULT_POOLSIZE,
+        help='If the value is not defined, we use the value defined by '
+             'requests.adapters.DEFAULT_POOLSIZE',
+    )
 ]
 
 
@@ -105,7 +114,9 @@ class GnocchiFetcher(fetcher.BaseFetcher):
 
         self._conn = gclient.Client(
             '1',
-            session_options={'auth': auth_plugin, 'verify': verify},
+            session=custom_session.create_custom_session(
+                {'auth': auth_plugin, 'verify': verify},
+                CONF.fetcher_gnocchi.http_pool_maxsize),
             adapter_options=adapter_options,
         )
 

releasenotes/notes/make-gnocchi-http-max-connections-pool-configurable-52c9f6617466ea30.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Adds a new configuration ``http_pool_maxsize`` that defines the maximum
+    size of Gnocchi's fetcher and collector HTTP connection pools.
+
+    The default value of this new configuration is defined by the ``requests``
+    library in the ``requests.adapters.DEFAULT_POOLSIZE`` global variable.