Add a Monasca fetcher

This adds a Monasca fetcher for CloudKitty.

Change-Id: I0bf5af21987878d1e078021b0a9fbd0cc0a37b9d
Story: 2006675
Task: 36949

cloudkitty/collector/monasca.py

@@ -14,8 +14,6 @@
 #    under the License.
 #
 from keystoneauth1 import loading as ks_loading
-from keystoneclient.v3 import client as ks_client
-from monascaclient import client as mclient
 from oslo_config import cfg
 from oslo_log import log as logging
 from voluptuous import All
@@ -25,10 +23,10 @@ from voluptuous import Required
 from voluptuous import Schema
 
 from cloudkitty import collector
+from cloudkitty.common import monasca_client as mon_client_utils
 from cloudkitty import dataframe
 from cloudkitty import utils as ck_utils
 
-
 LOG = logging.getLogger(__name__)
 
 MONASCA_API_VERSION = '2_0'
@@ -69,10 +67,6 @@ MONASCA_EXTRA_SCHEMA = {
 }
 
 
-class EndpointNotFound(Exception):
-    """Exception raised if the Monasca endpoint is not found"""
-
-
 class MonascaCollector(collector.BaseCollector):
     collector_name = 'monasca'
 
@@ -93,41 +87,8 @@ class MonascaCollector(collector.BaseCollector):
 
     def __init__(self, **kwargs):
         super(MonascaCollector, self).__init__(**kwargs)
-
+        self._conn = mon_client_utils.get_monasca_client(
-        self.auth = ks_loading.load_auth_from_conf_options(
+            CONF, COLLECTOR_MONASCA_OPTS)
-            CONF,
-            COLLECTOR_MONASCA_OPTS)
-        self.session = ks_loading.load_session_from_conf_options(
-            CONF,
-            COLLECTOR_MONASCA_OPTS,
-            auth=self.auth)
-        self.ks_client = ks_client.Client(
-            session=self.session,
-            interface=CONF.collector_monasca.interface,
-        )
-        self.mon_endpoint = self._get_monasca_endpoint()
-        if not self.mon_endpoint:
-            raise EndpointNotFound()
-        self._conn = mclient.Client(
-            api_version=MONASCA_API_VERSION,
-            session=self.session,
-            endpoint=self.mon_endpoint)
-
-    # NOTE(lukapeschke) This function should be removed as soon as the endpoint
-    # it no longer required by monascaclient
-    def _get_monasca_endpoint(self):
-        service_name = cfg.CONF.collector_monasca.monasca_service_name
-        endpoint_interface_type = cfg.CONF.collector_monasca.interface
-
-        service_list = self.ks_client.services.list(name=service_name)
-        if not service_list:
-            return None
-        mon_service = service_list[0]
-        endpoints = self.ks_client.endpoints.list(mon_service.id)
-        for endpoint in endpoints:
-            if endpoint.interface == endpoint_interface_type:
-                return endpoint.url
-        return None
 
     def _get_metadata(self, metric_name, conf):
         info = {}

cloudkitty/common/monasca_client.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+from keystoneauth1 import loading as ks_loading
+from keystoneclient.v3 import client as ks_client
+from monascaclient import client as mclient
+
+
+MONASCA_API_VERSION = '2_0'
+
+
+class EndpointNotFound(Exception):
+    """Exception raised if the Monasca endpoint is not found"""
+
+
+# NOTE(lukapeschke) This function should be removed as soon as the endpoint
+# it no longer required by monascaclient
+def get_monasca_endpoint(cfg, keystone_client):
+    service_name = cfg.monasca_service_name
+    endpoint_interface_type = cfg.interface
+
+    service_list = keystone_client.services.list(name=service_name)
+    if not service_list:
+        return None
+    mon_service = service_list[0]
+    endpoints = keystone_client.endpoints.list(mon_service.id)
+    for endpoint in endpoints:
+        if endpoint.interface == endpoint_interface_type:
+            return endpoint.url
+    return None
+
+
+def get_monasca_client(conf, conf_opts):
+    ks_auth = ks_loading.load_auth_from_conf_options(conf, conf_opts)
+    session = ks_loading.load_session_from_conf_options(
+        conf,
+        conf_opts,
+        auth=ks_auth)
+    keystone_client = ks_client.Client(
+        session=session,
+        interface=conf[conf_opts].interface)
+    mon_endpoint = get_monasca_endpoint(conf[conf_opts], keystone_client)
+    if not mon_endpoint:
+        raise EndpointNotFound()
+    return mclient.Client(
+        api_version=MONASCA_API_VERSION,
+        session=session,
+        endpoint=mon_endpoint)

cloudkitty/fetcher/monasca.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+from keystoneauth1 import loading as ks_loading
+from oslo_config import cfg
+
+from cloudkitty.common import monasca_client as mon_client_utils
+from cloudkitty import fetcher
+
+MONASCA_API_VERSION = '2_0'
+
+FETCHER_MONASCA_OPTS = 'fetcher_monasca'
+
+fetcher_monasca_opts = [
+    cfg.StrOpt('dimension_name',
+               default='project_id',
+               help='Monasca dimension from which scope_ids should be'
+               ' collected.'),
+    cfg.StrOpt('monasca_tenant_id',
+               default=None,
+               help='If specified, monasca client will use this ID instead of'
+               ' the default one.'),
+    cfg.StrOpt(
+        'monasca_service_name',
+        default='monasca',
+        help='Name of the Monasca service (defaults to monasca)',
+    ),
+    cfg.StrOpt(
+        'interface',
+        default='internal',
+        help='Endpoint URL type (defaults to internal).',
+    ),
+]
+
+CONF = cfg.CONF
+
+cfg.CONF.register_opts(fetcher_monasca_opts, FETCHER_MONASCA_OPTS)
+ks_loading.register_auth_conf_options(CONF, FETCHER_MONASCA_OPTS)
+ks_loading.register_session_conf_options(CONF, FETCHER_MONASCA_OPTS)
+
+
+class MonascaFetcher(fetcher.BaseFetcher):
+    """Monasca fetcher"""
+
+    name = 'monasca'
+
+    def __init__(self):
+        self._conn = mon_client_utils.get_monasca_client(CONF,
+                                                         FETCHER_MONASCA_OPTS)
+
+    def get_tenants(self):
+        kwargs = {
+            "tenant_id": CONF.fetcher_monasca.monasca_tenant_id,
+            "dimension_name": CONF.fetcher_monasca.dimension_name,
+        }
+        if kwargs['tenant_id'] is None:
+            del kwargs['tenant_id']
+        values = self._conn.metrics.list_dimension_values(**kwargs)
+        return [v['dimension_value'] for v in values]

cloudkitty/tests/collectors/test_monasca.py

@@ -44,8 +44,8 @@ class MonascaCollectorTest(tests.TestCase):
             }
         }
         with mock.patch(
-                'cloudkitty.collector.monasca.'
+                'cloudkitty.common.monasca_client.'
-                'MonascaCollector._get_monasca_endpoint',
+                'get_monasca_endpoint',
                 return_value='http://noop'):
             self.collector = mon_collector.MonascaCollector(
                 period=3600,

cloudkitty/tests/fetchers/test_monasca.py

@@ -0,0 +1,48 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+from unittest import mock
+
+from cloudkitty.fetcher import monasca as mon_fetcher
+from cloudkitty import tests
+
+
+class MonascaFetcherTest(tests.TestCase):
+
+    def setUp(self):
+        super(MonascaFetcherTest, self).setUp()
+        self.conf.set_override('dimension_name', 'dimension_name_test',
+                               'fetcher_monasca')
+        self.conf.set_override('monasca_tenant_id', 'this_is_definitely_a_uid',
+                               'fetcher_monasca')
+        self.conf.set_override('monasca_service_name', 'monasca-api-test',
+                               'fetcher_monasca')
+        self.conf.set_override('interface', 'interface-test',
+                               'fetcher_monasca')
+
+        with mock.patch(
+                'cloudkitty.common.monasca_client.'
+                'get_monasca_endpoint',
+                return_value='http://noop'):
+            self.fetcher = mon_fetcher.MonascaFetcher()
+
+    def test_get_tenants(self):
+        with mock.patch.object(self.fetcher._conn.metrics,
+                               'list_dimension_values') as m:
+            self.fetcher.get_tenants()
+            m.assert_called_once_with(
+                tenant_id='this_is_definitely_a_uid',
+                dimension_name='dimension_name_test',
+            )

doc/source/admin/architecture.rst

@@ -63,7 +63,7 @@ configured in CloudKitty's configuration file.
 Fetcher
 =======
 
-Four fetchers are available in cloudkitty:
+Five fetchers are available in CloudKitty:
 
 * The ``keystone`` fetcher retrieves a list of projects on which the
   cloudkitty user has the ``rating`` role from Keystone.
@@ -73,6 +73,9 @@ Four fetchers are available in cloudkitty:
   discover new projects from Gnocchi when it is used with OpenStack. It can be
   used in an OpenStack context or with a standalone Gnocchi deployment.
 
+* The ``monasca`` fetcher retrieves from `Monasca`_ all values from a
+  configurable metric dimension (``project_id`` by default).
+
 * The ``prometheus`` fetcher works in a similar way to the Gnocchi fetcher,
   which allows to discover scopes from `Prometheus`_.
 

doc/source/admin/configuration/fetcher.rst

@@ -60,6 +60,20 @@ fetcher using regular Keystone authentication options as found here:
 :doc:`configuration`.
 
 
+Monasca
+-------
+
+Section ``fetcher_monasca``.
+
+* ``dimension_name``: Monasca dimension from which scope_ids is collected.
+
+* ``monasca_tenant_id``: If specified, monasca client will use this ID
+  instead of the default one.
+
+* ``monasca_service_name``: Name of the Monasca service (defaults to monasca).
+
+* ``interface``: Endpoint URL type (defaults to internal).
+
 Prometheus
 ----------
 

releasenotes/notes/monasca-fetcher-2ea866f873ab5336.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Adds a Monasca fetcher retrieving scopes from Monasca dimensions. See the
+    `fetcher documentation
+    <https://docs.openstack.org/cloudkitty/latest/admin/configuration/fetcher.html#monasca>`__
+    for more details.

setup.cfg

@@ -56,6 +56,7 @@ cloudkitty.fetchers =
     source = cloudkitty.fetcher.source:SourceFetcher
     gnocchi = cloudkitty.fetcher.gnocchi:GnocchiFetcher
     prometheus = cloudkitty.fetcher.prometheus:PrometheusFetcher
+    monasca = cloudkitty.fetcher.monasca:MonascaFetcher
 
 cloudkitty.rating.processors =
     noop = cloudkitty.rating.noop:Noop