Make multiple attempts to obtain gnocchiclient

As of now a single attempt is made to obtain gnocchiclient, if
this request fails due any intermittent issue, ceilometer fails to
load gnocchi publisher.

This change uses tenacity to retry fetching gnocchiclient if it
fails with keystone exceptions "ServiceUnavailable",
"DiscoveryFailure" or "ConnectTimeout" before assuming gnocchi service  is not available.

Upon restarting ceilometer services, "event_pipeline.yaml" definitions
file is logged during info mode which is unnecessary . This config
file must be logged only when debug is enabled.

Resolves: rhbz#2167428
Change-Id: Ia223b6518948765a74bb920829b6f76c354a3919

ceilometer/declarative.py

@@ -199,5 +199,5 @@ def load_definitions(conf, defaults, config_file, fallback_file=None):
                   "Using default config.")
         definition_cfg = defaults
 
-    LOG.info("Definitions: %s", definition_cfg)
+    LOG.debug("Definitions: %s", definition_cfg)
     return definition_cfg

ceilometer/publisher/gnocchi.py

@@ -18,6 +18,7 @@ import itertools
 import json
 import operator
 import pkg_resources
+import tenacity
 import threading
 
 from gnocchiclient import exceptions as gnocchi_exc
@@ -213,13 +214,28 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
         self._gnocchi_project_id_lock = threading.Lock()
         self._gnocchi_resource_lock = LockedDefaultDict(threading.Lock)
 
-        self._gnocchi = gnocchi_client.get_gnocchiclient(
-            conf, request_timeout=timeout)
+        try:
+            self._gnocchi = self._get_gnocchi_client(conf, timeout)
+        except tenacity.RetryError as e:
+            raise e.last_attempt._exception from None
+
         self._already_logged_event_types = set()
         self._already_logged_metric_names = set()
 
         self._already_configured_archive_policies = False
 
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(10),
+        wait=tenacity.wait_fixed(5),
+        retry=(
+            tenacity.retry_if_exception_type(ka_exceptions.ServiceUnavailable)
+            | tenacity.retry_if_exception_type(ka_exceptions.DiscoveryFailure)
+            | tenacity.retry_if_exception_type(ka_exceptions.ConnectTimeout)
+        ),
+        reraise=False)
+    def _get_gnocchi_client(self, conf, timeout):
+        return gnocchi_client.get_gnocchiclient(conf, request_timeout=timeout)
+
     @staticmethod
     def _load_definitions(conf, archive_policy_override,
                           resources_definition_file):

ceilometer/tests/unit/publisher/test_gnocchi.py

@@ -343,6 +343,15 @@ class PublisherTest(base.BaseTestCase):
             'Filtered project [service] not found in keystone, ignoring the '
             'filter_project option')
 
+    @mock.patch('ceilometer.publisher.gnocchi.GnocchiPublisher'
+                '._get_gnocchi_client')
+    def test_get_gnocchi_client(self, gnocchi_cli):
+        url = netutils.urlsplit("gnocchi://")
+        gnocchi_cli.side_effect = ka_exceptions.DiscoveryFailure
+        cfg = self.conf.conf
+        publisher = gnocchi.GnocchiPublisher
+        self.assertRaises(ka_exceptions.DiscoveryFailure, publisher, cfg, url)
+
     def test_activity_filter_match_swift_event(self):
         self.samples[0].name = 'storage.objects.outgoing.bytes'
         self.samples[0].resource_id = 'a2d42c23-d518-46b6-96ab-3fba2e146859'