Retrieve prometheus information from keystone

This adds the possibility to retrieve prometheus or aetos endpoint
information from keystone. Aetos is expected to have the endpoint
registered (Aetos's devstack plugin already does this) and admins
can register plain prometheus in Keystone with a simple command.

Previous functionality with using /etc/openstack/prometheus.yaml or
env variables is kept unchanged.

Change-Id: I20eb5858244f1202ab8bc1fa26bb46b41d927ac0

observabilityclient/tests/unit/test_utils.py

@@ -15,6 +15,8 @@
 import os
 from unittest import mock
 
+from keystoneauth1 import adapter
+from keystoneauth1 import session
 import testtools
 
 from observabilityclient import prometheus_client
@@ -99,6 +101,62 @@ class GetPrometheusClientTest(testtools.TestCase):
             self.assertRaises(metric_utils.ConfigurationError,
                               metric_utils.get_prometheus_client)
 
+    def test_get_prometheus_client_from_keystone_http(self):
+        prometheus_endpoint = "http://localhost:1234/prometheus"
+        keystone_session = session.Session()
+        with mock.patch.dict(os.environ, {}), \
+                mock.patch.object(metric_utils, 'get_config_file',
+                                  return_value=None), \
+                mock.patch.object(adapter.Adapter, 'get_endpoint',
+                                  return_value=prometheus_endpoint), \
+                mock.patch.object(prometheus_client.PrometheusAPIClient,
+                                  "__init__", return_value=None) as init_m, \
+                mock.patch.object(prometheus_client.PrometheusAPIClient,
+                                  "set_ca_cert") as ca_m:
+            metric_utils.get_prometheus_client(keystone_session)
+        init_m.assert_called_with(
+            "localhost:1234", keystone_session, "prometheus"
+        )
+        ca_m.assert_not_called()
+
+    def test_get_prometheus_client_from_keystone_https(self):
+        prometheus_endpoint = "https://localhost:1234/prometheus"
+        keystone_session = session.Session()
+        with mock.patch.dict(os.environ, {}), \
+                mock.patch.object(metric_utils, 'get_config_file',
+                                  return_value=None), \
+                mock.patch.object(adapter.Adapter, 'get_endpoint',
+                                  return_value=prometheus_endpoint), \
+                mock.patch.object(prometheus_client.PrometheusAPIClient,
+                                  "__init__", return_value=None) as init_m, \
+                mock.patch.object(prometheus_client.PrometheusAPIClient,
+                                  "set_ca_cert") as ca_m:
+            metric_utils.get_prometheus_client(keystone_session)
+        init_m.assert_called_with(
+            "localhost:1234", keystone_session, "prometheus"
+        )
+        ca_m.assert_called_with(True)
+
+    def test_get_prometheus_client_from_keystone_custom_ca(self):
+        prometheus_endpoint = "https://localhost:1234/prometheus"
+        keystone_session = session.Session()
+        config_data = 'ca_cert: "ca/path"'
+        config_file = mock.mock_open(read_data=config_data)("name", 'r')
+        with mock.patch.dict(os.environ, {}), \
+                mock.patch.object(metric_utils, 'get_config_file',
+                                  return_value=config_file), \
+                mock.patch.object(adapter.Adapter, 'get_endpoint',
+                                  return_value=prometheus_endpoint), \
+                mock.patch.object(prometheus_client.PrometheusAPIClient,
+                                  "__init__", return_value=None) as init_m, \
+                mock.patch.object(prometheus_client.PrometheusAPIClient,
+                                  "set_ca_cert") as ca_m:
+            metric_utils.get_prometheus_client(keystone_session)
+        init_m.assert_called_with(
+            "localhost:1234", keystone_session, "prometheus"
+        )
+        ca_m.assert_called_with("ca/path")
+
 
 class FormatLabelsTest(testtools.TestCase):
     def setUp(self):

observabilityclient/utils/metric_utils.py

@@ -14,7 +14,10 @@
 
 import logging
 import os
+from urllib import parse
 
+from keystoneauth1 import adapter
+from keystoneauth1.exceptions import catalog as keystone_exception
 import yaml
 
 from observabilityclient.prometheus_client import PrometheusAPIClient
@@ -45,7 +48,7 @@ def get_config_file():
     return None
 
 
-def get_prometheus_client(session=None):
+def get_prometheus_client(session=None, adapter_options={}):
     host = None
     port = None
     ca_cert = None
@@ -63,6 +66,28 @@ def get_prometheus_client(session=None):
             root_path = conf['root_path']
         conf_file.close()
 
+    if session is not None and (host is None or port is None):
+        try:
+            endpoint = adapter.Adapter(
+                session=session, **adapter_options
+            ).get_endpoint()
+            parsed_url = parse.urlparse(endpoint)
+            host = parsed_url.hostname
+            port = parsed_url.port if parsed_url.port is not None else 80
+            root_path = parsed_url.path.strip('/')
+            if parsed_url.scheme == "https" and ca_cert is None:
+                # NOTE(jwysogla): Use the default CA certs if the scheme
+                # is https, but keep the original value if already set,
+                # so that a custom certificate can be set in the config
+                # file, while the endpoint is retrieved from keystone.
+                ca_cert = True
+        except keystone_exception.EndpointNotFound:
+            # NOTE(jwysogla): Don't do anything here. It's still possible
+            # to get the correct endpoint configuration from the env vars.
+            # If that doesn't work, the same error message is part of the
+            # exception raised below.
+            pass
+
     # NOTE(jwysogla): We allow to overide the prometheus.yaml by
     #                 the environment variables
     if 'PROMETHEUS_HOST' in os.environ:
@@ -75,7 +100,8 @@ def get_prometheus_client(session=None):
         root_path = os.environ['PROMETHEUS_ROOT_PATH']
     if host is None or port is None:
         raise ConfigurationError("Can't find prometheus host and "
-                                 "port configuration.")
+                                 "port configuration and endpoint for service"
+                                 "prometheus not found.")
     client = PrometheusAPIClient(f"{host}:{port}", session, root_path)
     if ca_cert is not None:
         client.set_ca_cert(ca_cert)

observabilityclient/v1/client.py

@@ -28,7 +28,7 @@ class Client(object):
         session_options = session_options or {}
         adapter_options = adapter_options or {}
 
-        adapter_options.setdefault('service_type', "metric")
+        adapter_options.setdefault('service_type', "prometheus")
 
         if session is None:
             session = keystoneauth1.session.Session(**session_options)
@@ -38,7 +38,9 @@ class Client(object):
 
         self.session = session
 
-        self.prometheus_client = get_prometheus_client(session)
+        self.prometheus_client = get_prometheus_client(
+            session, adapter_options
+        )
         self.query = python_api.QueryManager(self)
         self.rbac = rbac.PromQLRbac(
             self.prometheus_client,