verify gnocchi connection before processing

currently we create a client on initialisation but never verify that
the client has a connection. this creates a lot of errors in logs
because gnocchi isn't started yet.

this patch verfies the connection before it begins processing data by
calling gnocchi on initialisation.

DocImpact
Closes-Bug: #1489949
Change-Id: Ie3b7d966f068611e3fb60089b724f1a4c02f771e

ceilometer/dispatcher/__init__.py

@@ -37,6 +37,19 @@ OPTS = [
 ]
 cfg.CONF.register_opts(OPTS)
 
+STORAGE_OPTS = [
+    cfg.IntOpt('max_retries',
+               default=10,
+               deprecated_group='database',
+               help='Maximum number of connection retries during startup. '
+                    'Set to -1 to specify an infinite retry count.'),
+    cfg.IntOpt('retry_interval',
+               default=10,
+               deprecated_group='database',
+               help='Interval (in seconds) between retries of connection.'),
+]
+cfg.CONF.register_opts(STORAGE_OPTS, group='storage')
+
 
 def _load_dispatcher_manager(dispatcher_type):
     namespace = 'ceilometer.dispatcher.%s' % dispatcher_type

ceilometer/dispatcher/gnocchi.py

@@ -24,6 +24,7 @@ from keystoneauth1 import session as ka_session
 from oslo_config import cfg
 from oslo_log import log
 import requests
+import retrying
 import six
 from stevedore import extension
 
@@ -197,6 +198,20 @@ class GnocchiDispatcher(dispatcher.MeterDispatcherBase):
         self._gnocchi_resource_lock = threading.Lock()
 
         self._gnocchi = GnocchiClient(conf)
+        # Convert retry_interval secs to msecs for retry decorator
+        retries = conf.storage.max_retries
+
+        @retrying.retry(wait_fixed=conf.storage.retry_interval * 1000,
+                        stop_max_attempt_number=(retries if retries >= 0
+                                                 else None))
+        def _get_connection():
+            self._gnocchi.capabilities.list()
+
+        try:
+            _get_connection()
+        except Exception:
+            LOG.error(_LE('Failed to connect to Gnocchi.'))
+            raise
 
     @staticmethod
     def _get_config_file(conf, config_file):

ceilometer/opts.py

@@ -122,6 +122,7 @@ def list_opts():
                          ceilometer.nova_client.SERVICE_OPTS,
                          ceilometer.objectstore.rgw.SERVICE_OPTS,
                          ceilometer.objectstore.swift.SERVICE_OPTS,)),
+        ('storage', ceilometer.dispatcher.STORAGE_OPTS),
         ('vmware', ceilometer.compute.virt.vmware.inspector.OPTS),
         ('xenapi', ceilometer.compute.virt.xenapi.inspector.OPTS),
     ]

ceilometer/tests/unit/dispatcher/test_gnocchi.py

@@ -35,6 +35,7 @@ from ceilometer.tests import base
 load_tests = testscenarios.load_tests_apply_scenarios
 
 
+@mock.patch('gnocchiclient.v1.client.Client', mock.Mock())
 class DispatcherTest(base.BaseTestCase):
 
     def setUp(self):
@@ -313,8 +314,11 @@ class DispatcherWorkflowTest(base.BaseTestCase,
         resource_id = self.sample['resource_id']  # .replace("/", "%2F"),
         metric_name = self.sample['counter_name']
 
-        expected_calls = [mock.call.metric.add_measures(
+        expected_calls = [
-            metric_name, self.measures_attributes, resource_id)]
+            mock.call.capabilities.list(),
+            mock.call.metric.add_measures(metric_name,
+                                          self.measures_attributes,
+                                          resource_id)]
 
         add_measures_side_effect = []