Adding two options in fetcher_keystone

This patch adds two options in fetcher_keystone to filter which tenants
should be rated:

ignore_disabled_tenants (Default=False)
ignore_rating_role (Default=False)

In our case we currently have 2k projects (growing) and we want to rate
all active projects, so checking the role rating is useless and consumes
resources for nothing. Besides, cloudkitty rates projects regardless if
there are enabled or disabled which is also useless and consumes
resources in our case.

Change-Id: I6479d76c367dc4217bce4de9c3db41c4612f0397

cloudkitty/fetcher/keystone.py

@@ -19,10 +19,10 @@ from keystoneclient import client as kclient
 from keystoneclient import discover
 from keystoneclient import exceptions
 from oslo_config import cfg
+from oslo_log import log as logging
 
 from cloudkitty import fetcher
 
-
 FETCHER_KEYSTONE_OPTS = 'fetcher_keystone'
 
 fetcher_keystone_opts = [
@@ -31,6 +31,16 @@ fetcher_keystone_opts = [
         default='3',
         help='Keystone version to use.',
     ),
+    cfg.BoolOpt(
+        'ignore_rating_role',
+        default=False,
+        help='Skip rating role check for cloudkitty user',
+    ),
+    cfg.BoolOpt(
+        'ignore_disabled_tenants',
+        default=False,
+        help='Stop rating disabled tenants',
+    ),
 ]
 
 ks_loading.register_session_conf_options(cfg.CONF, FETCHER_KEYSTONE_OPTS)
@@ -39,6 +49,8 @@ cfg.CONF.register_opts(fetcher_keystone_opts, FETCHER_KEYSTONE_OPTS)
 
 CONF = cfg.CONF
 
+LOG = logging.getLogger(__name__)
+
 
 class KeystoneFetcher(fetcher.BaseFetcher):
     """Keystone tenants fetcher."""
@@ -73,10 +85,21 @@ class KeystoneFetcher(fetcher.BaseFetcher):
         tenant_attr, tenants_attr, role_func = auth_version_mapping
         tenant_list = getattr(self.admin_ks, tenants_attr).list()
         my_user_id = self.session.get_user_id()
+        ignore_rating_role = CONF.fetcher_keystone.ignore_rating_role
+        ignore_disabled_tenants = CONF.fetcher_keystone.ignore_disabled_tenants
+        LOG.debug('Total number of tenants : %s', len(tenant_list))
         for tenant in tenant_list[:]:
-            roles = getattr(self.admin_ks.roles, role_func)(
-                **{'user': my_user_id,
-                   tenant_attr: tenant})
-            if 'rating' not in [role.name for role in roles]:
-                tenant_list.remove(tenant)
+            if ignore_disabled_tenants:
+                if not tenant.enabled:
+                    tenant_list.remove(tenant)
+                    LOG.debug('Disabled tenant name %s with id %s skipped.',
+                              tenant.name, tenant.id)
+                    continue
+            if not ignore_rating_role:
+                roles = getattr(self.admin_ks.roles, role_func)(
+                    **{'user': my_user_id,
+                       tenant_attr: tenant})
+                if 'rating' not in [role.name for role in roles]:
+                    tenant_list.remove(tenant)
+        LOG.debug('Number of tenants to rate : %s', len(tenant_list))
         return [tenant.id for tenant in tenant_list]

doc/source/admin/configuration/fetcher.rst

@@ -59,6 +59,13 @@ If ``auth_section`` option is not defined then you can configure Keystone
 fetcher using regular Keystone authentication options as found here:
 :doc:`configuration`.
 
+* ``ignore_rating_role``: if set to true, the Keystone fetcher will not check
+  if a project has the rating role; thus, CloudKitty will execute rating for
+  every project it finds. Defaults to false.
+
+* ``ignore_disabled_tenants``: if set to true, Cloudkitty will not rate
+  projects that are disabled in Keystone. Defaults to false.
+
 
 Monasca
 -------

releasenotes/notes/ignore_disabled_tenants-and-ignore_rating_role-dfe542a0cafd412e.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Two new options ``ignore_disabled_tenants`` and ``ignore_rating_role``
+    were added in the ``fetcher_keystone`` section. ``ignore_disabled_tenants``
+    skips disabled tenants when doing the rating. ``ignore_rating_role``
+    rates everyone, without reading the rating role for each project, which
+    can be resource consuming.