Add a gnocchi fetcher

This adds a gnocchi fetcher, which allows to dynamically retrieve available
scopes. Documentation will be extended once #592329 is merged.

Story: 2003652
Task: 26065

Change-Id: I9095008ea00dde99cbacb8f716d2d6b600fd4931

cloudkitty/common/config.py

@@ -21,6 +21,7 @@ import cloudkitty.collector.gnocchi
 import cloudkitty.collector.monasca
 import cloudkitty.config
 import cloudkitty.fetcher
+import cloudkitty.fetcher.gnocchi
 import cloudkitty.fetcher.keystone
 import cloudkitty.fetcher.source
 import cloudkitty.orchestrator
@@ -43,6 +44,9 @@ _opts = [
     ('gnocchi_collector', list(itertools.chain(
         cloudkitty.collector.gnocchi.gcollector_opts,
         cloudkitty.collector.gnocchi.gnocchi_collector_opts))),
+    ('fetcher_gnocchi', list(itertools.chain(
+        cloudkitty.fetcher.gnocchi.gfetcher_opts,
+        cloudkitty.fetcher.gnocchi.fetcher_gnocchi_opts))),
     ('keystone_fetcher', list(itertools.chain(
         cloudkitty.fetcher.keystone.keystone_fetcher_opts,
         cloudkitty.fetcher.keystone.keystone_common_opts))),

cloudkitty/fetcher/gnocchi.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Luka Peschke
+#
+from gnocchiclient import auth as gauth
+from gnocchiclient import client as gclient
+from keystoneauth1 import loading as ks_loading
+from oslo_config import cfg
+from oslo_log import log
+
+from cloudkitty import fetcher
+
+
+LOG = log.getLogger(__name__)
+
+FETCHER_GNOCCHI_OPTS = 'fetcher_gnocchi'
+
+fetcher_gnocchi_opts = ks_loading.get_auth_common_conf_options()
+gfetcher_opts = [
+    cfg.StrOpt('scope_attribute',
+               default='project_id',
+               help='Attribute from which scope_ids should be collected.'),
+    cfg.ListOpt('resource_types',
+                default=['generic'],
+                help='List of gnocchi resource types. All if left blank'),
+    cfg.StrOpt(
+        'gnocchi_auth_type',
+        default='keystone',
+        choices=['keystone', 'basic'],
+        help='Gnocchi auth type (keystone or basic). Keystone credentials '
+        'can be specified through the "auth_section" parameter',
+    ),
+    cfg.StrOpt(
+        'gnocchi_user',
+        default='',
+        help='Gnocchi user (for basic auth only)',
+    ),
+    cfg.StrOpt(
+        'gnocchi_endpoint',
+        default='',
+        help='Gnocchi endpoint (for basic auth only)',
+    ),
+    cfg.StrOpt(
+        'interface',
+        default='internalURL',
+        help='Endpoint URL type (for keystone auth only)',
+    ),
+    cfg.StrOpt(
+        'region_name',
+        default='RegionOne',
+        help='Region Name'),
+]
+
+
+cfg.CONF.register_opts(fetcher_gnocchi_opts, FETCHER_GNOCCHI_OPTS)
+cfg.CONF.register_opts(gfetcher_opts, FETCHER_GNOCCHI_OPTS)
+ks_loading.register_session_conf_options(
+    cfg.CONF,
+    FETCHER_GNOCCHI_OPTS)
+ks_loading.register_auth_conf_options(
+    cfg.CONF,
+    FETCHER_GNOCCHI_OPTS)
+CONF = cfg.CONF
+
+
+class GnocchiFetcher(fetcher.BaseFetcher):
+    """Gnocchi scope_id fetcher."""
+
+    name = 'gnocchi'
+
+    def __init__(self):
+        super(GnocchiFetcher, self).__init__()
+        adapter_options = {'connect_retries': 3}
+        if CONF.fetcher_gnocchi.gnocchi_auth_type == 'keystone':
+            auth_plugin = ks_loading.load_auth_from_conf_options(
+                CONF,
+                FETCHER_GNOCCHI_OPTS,
+            )
+            adapter_options['interface'] = CONF.fetcher_gnocchi.interface
+        else:
+            auth_plugin = gauth.GnocchiBasicPlugin(
+                user=CONF.fetcher_gnocchi.gnocchi_user,
+                endpoint=CONF.fetcher_gnocchi.gnocchi_endpoint,
+            )
+        adapter_options['region_name'] = CONF.fetcher_gnocchi.region_name
+
+        self._conn = gclient.Client(
+            '1',
+            session_options={'auth': auth_plugin},
+            adapter_options=adapter_options,
+        )
+
+    def get_tenants(self):
+        resources = []
+        resource_types = CONF.fetcher_gnocchi.resource_types
+        for resource_type in resource_types:
+            marker = None
+            while True:
+                resources_chunk = self._conn.resource.list(
+                    resource_type=resource_type,
+                    marker=marker,
+                    details=True)
+                if len(resources_chunk) < 1 or (
+                        len(resources) == 1 and resources[0]['id'] == marker):
+                    break
+                resources += resources_chunk
+                marker = resources_chunk[-1]['id']
+
+        scope_attribute = CONF.fetcher_gnocchi.scope_attribute
+        scope_ids = [
+            resource.get(scope_attribute, None) for resource in resources]
+        scope_ids = [s_id for s_id in scope_ids if s_id]
+        # Returning unique ids
+        return list(set(scope_ids))

releasenotes/notes/add-gnocchi-fetcher-b8a6e2ea49fcfec5.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    A fetcher for gnocchi has been added, allowing dynamic scope/project
+    discovery.

setup.cfg

@@ -55,6 +55,7 @@ cloudkitty.fetchers =
     fake = cloudkitty.fetcher.fake:FakeFetcher
     keystone = cloudkitty.fetcher.keystone:KeystoneFetcher
     source = cloudkitty.fetcher.source:SourceFetcher
+    gnocchi = cloudkitty.fetcher.gnocchi:GnocchiFetcher
 
 cloudkitty.transformers =
     CloudKittyFormatTransformer = cloudkitty.transformer.format:CloudKittyFormatTransformer