From 152aa1af01769841956da1bc0225bde88b80af08 Mon Sep 17 00:00:00 2001
From: gordon chung <gord@live.ca>
Date: Fri, 4 Dec 2015 13:11:58 -0500
Subject: [PATCH] verify gnocchi connection before processing

currently we create a client on initialisation but never verify that
the client has a connection. this creates a lot of errors in logs
because gnocchi isn't started yet.

this patch verfies the connection before it begins processing data by
calling gnocchi on initialisation.

DocImpact
Closes-Bug: #1489949
Change-Id: Ie3b7d966f068611e3fb60089b724f1a4c02f771e
---
 ceilometer/dispatcher/__init__.py                | 13 +++++++++++++
 ceilometer/dispatcher/gnocchi.py                 | 15 +++++++++++++++
 ceilometer/opts.py                               |  1 +
 ceilometer/tests/unit/dispatcher/test_gnocchi.py |  8 ++++++--
 4 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/ceilometer/dispatcher/__init__.py b/ceilometer/dispatcher/__init__.py
index 8e56688d..40a1e122 100644
--- a/ceilometer/dispatcher/__init__.py
+++ b/ceilometer/dispatcher/__init__.py
@@ -37,6 +37,19 @@ OPTS = [
 ]
 cfg.CONF.register_opts(OPTS)
 
+STORAGE_OPTS = [
+    cfg.IntOpt('max_retries',
+               default=10,
+               deprecated_group='database',
+               help='Maximum number of connection retries during startup. '
+                    'Set to -1 to specify an infinite retry count.'),
+    cfg.IntOpt('retry_interval',
+               default=10,
+               deprecated_group='database',
+               help='Interval (in seconds) between retries of connection.'),
+]
+cfg.CONF.register_opts(STORAGE_OPTS, group='storage')
+
 
 def _load_dispatcher_manager(dispatcher_type):
     namespace = 'ceilometer.dispatcher.%s' % dispatcher_type
diff --git a/ceilometer/dispatcher/gnocchi.py b/ceilometer/dispatcher/gnocchi.py
index 3c4fea63..20b7652a 100644
--- a/ceilometer/dispatcher/gnocchi.py
+++ b/ceilometer/dispatcher/gnocchi.py
@@ -24,6 +24,7 @@ from keystoneauth1 import session as ka_session
 from oslo_config import cfg
 from oslo_log import log
 import requests
+import retrying
 import six
 from stevedore import extension
 
@@ -197,6 +198,20 @@ class GnocchiDispatcher(dispatcher.MeterDispatcherBase):
         self._gnocchi_resource_lock = threading.Lock()
 
         self._gnocchi = GnocchiClient(conf)
+        # Convert retry_interval secs to msecs for retry decorator
+        retries = conf.storage.max_retries
+
+        @retrying.retry(wait_fixed=conf.storage.retry_interval * 1000,
+                        stop_max_attempt_number=(retries if retries >= 0
+                                                 else None))
+        def _get_connection():
+            self._gnocchi.capabilities.list()
+
+        try:
+            _get_connection()
+        except Exception:
+            LOG.error(_LE('Failed to connect to Gnocchi.'))
+            raise
 
     @staticmethod
     def _get_config_file(conf, config_file):
diff --git a/ceilometer/opts.py b/ceilometer/opts.py
index 1902f7ae..384c2565 100644
--- a/ceilometer/opts.py
+++ b/ceilometer/opts.py
@@ -122,6 +122,7 @@ def list_opts():
                          ceilometer.nova_client.SERVICE_OPTS,
                          ceilometer.objectstore.rgw.SERVICE_OPTS,
                          ceilometer.objectstore.swift.SERVICE_OPTS,)),
+        ('storage', ceilometer.dispatcher.STORAGE_OPTS),
         ('vmware', ceilometer.compute.virt.vmware.inspector.OPTS),
         ('xenapi', ceilometer.compute.virt.xenapi.inspector.OPTS),
     ]
diff --git a/ceilometer/tests/unit/dispatcher/test_gnocchi.py b/ceilometer/tests/unit/dispatcher/test_gnocchi.py
index e6c51533..4f902b86 100644
--- a/ceilometer/tests/unit/dispatcher/test_gnocchi.py
+++ b/ceilometer/tests/unit/dispatcher/test_gnocchi.py
@@ -35,6 +35,7 @@ from ceilometer.tests import base
 load_tests = testscenarios.load_tests_apply_scenarios
 
 
+@mock.patch('gnocchiclient.v1.client.Client', mock.Mock())
 class DispatcherTest(base.BaseTestCase):
 
     def setUp(self):
@@ -313,8 +314,11 @@ class DispatcherWorkflowTest(base.BaseTestCase,
         resource_id = self.sample['resource_id']  # .replace("/", "%2F"),
         metric_name = self.sample['counter_name']
 
-        expected_calls = [mock.call.metric.add_measures(
-            metric_name, self.measures_attributes, resource_id)]
+        expected_calls = [
+            mock.call.capabilities.list(),
+            mock.call.metric.add_measures(metric_name,
+                                          self.measures_attributes,
+                                          resource_id)]
 
         add_measures_side_effect = []