From 79454d6b22787627ae6239aa7b2707101ba30212 Mon Sep 17 00:00:00 2001 From: Yadnesh Kulkarni Date: Mon, 1 Aug 2022 10:42:02 +0000 Subject: [PATCH] Add user/project names to polled samples Project and user names would be first fetched from cache, if not found, they will be requested from keystone and then cached. Using cache will significanlty reduce the number of calls made to keystone. If ceilometer is configured with no caching backend, the results will always be fetched by querying request to keystone. A new config option, `tenant_name_discovery` is introduced to operate this feature. This feature is optional and is disabled by default. No attempts to identify names will be made if uuids are found to be `None`. Signed-off-by: Yadnesh Kulkarni Change-Id: Iee5dbf09a1fd3ac571746fc66d2683eb8e6a1b27 --- ceilometer/cache_utils.py | 53 ++++++++++++++++ ceilometer/polling/manager.py | 63 ++++++++++++++++++- ceilometer/publisher/utils.py | 2 + ceilometer/sample.py | 5 +- ceilometer/tests/unit/polling/test_manager.py | 11 ++++ ...enant-name-discovery-668260bb4b2b0e8c.yaml | 6 ++ 6 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 ceilometer/cache_utils.py create mode 100644 releasenotes/notes/add-tenant-name-discovery-668260bb4b2b0e8c.yaml diff --git a/ceilometer/cache_utils.py b/ceilometer/cache_utils.py new file mode 100644 index 0000000000..55a9e2632c --- /dev/null +++ b/ceilometer/cache_utils.py @@ -0,0 +1,53 @@ +# +# Copyright 2022 Red Hat, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Simple wrapper for oslo_cache.""" + + +from oslo_cache import core as cache + + +class CacheClient(object): + def __init__(self, region): + self.region = region + + def get(self, key): + value = self.region.get(key) + if value == cache.NO_VALUE: + return None + return value + + def set(self, key, value): + return self.region.set(key, value) + + def delete(self, key): + return self.region.delete(key) + + +def get_client(conf, expiration_time=0): + cache.configure(conf) + if conf.cache.enabled: + return CacheClient(_get_default_cache_region( + conf, + expiration_time=expiration_time + )) + + +def _get_default_cache_region(conf, expiration_time): + region = cache.create_region() + if expiration_time != 0: + conf.cache.expiration_time = expiration_time + cache.configure_cache_region(conf, region) + return region diff --git a/ceilometer/polling/manager.py b/ceilometer/polling/manager.py index 6b9289d1d9..e5776a3e2d 100644 --- a/ceilometer/polling/manager.py +++ b/ceilometer/polling/manager.py @@ -35,6 +35,7 @@ from tooz import coordination from urllib import parse as urlparse from ceilometer import agent +from ceilometer import cache_utils from ceilometer import declarative from ceilometer import keystone_client from ceilometer import messaging @@ -45,6 +46,8 @@ from ceilometer import utils LOG = log.getLogger(__name__) +CACHE_DURATION = 3600 + POLLING_OPTS = [ cfg.StrOpt('cfg_file', default="polling.yaml", @@ -64,7 +67,18 @@ POLLING_OPTS = [ cfg.MultiStrOpt('pollsters_definitions_dirs', default=["/etc/ceilometer/pollsters.d"], help="List of directories with YAML files used " - "to created pollsters.") + "to created pollsters."), + cfg.BoolOpt('tenant_name_discovery', + default=False, + help="Identify project and user names from polled samples" + "By default, collecting these values is disabled due" + "to the fact that it could overwhelm keystone service" + "with lots of continuous requests depending upon the" + "number of projects, users and samples polled from" + "the environment. While using this feature, it is" + "recommended that ceilometer be configured with a" + "caching backend to reduce the number of calls" + "made to keystone"), ] @@ -138,11 +152,39 @@ class PollingTask(object): self._telemetry_secret = self.manager.conf.publisher.telemetry_secret + self.ks_client = self.manager.keystone + + self.cache_client = cache_utils.get_client( + self.manager.conf, + expiration_time=CACHE_DURATION + ) + def add(self, pollster, source): self.pollster_matches[source.name].add(pollster) key = Resources.key(source.name, pollster) self.resources[key].setup(source) + def resolve_uuid_from_cache(self, attr, uuid): + if self.cache_client: + name = self.cache_client.get(uuid) + if name: + return name + name = self.resolve_uuid_from_keystone(attr, uuid) + self.cache_client.set(uuid, name) + return name + + # Retrieve project and user names from Keystone only + # if ceilometer doesn't have a caching backend + return self.resolve_uuid_from_keystone(attr, uuid) + + def resolve_uuid_from_keystone(self, attr, uuid): + try: + return getattr(self.ks_client, attr).get(uuid).name + except AttributeError as e: + LOG.warning("Found '%s' while resolving uuid %s to name", e, uuid) + except ka_exceptions.NotFound as e: + LOG.warning(e.message) + def poll_and_notify(self): """Polling sample and notify.""" cache = {} @@ -194,6 +236,25 @@ class PollingTask(object): for sample in samples: # Note(yuywz): Unify the timestamp of polled samples sample.set_timestamp(polling_timestamp) + + if self.manager.conf.tenant_name_discovery: + + # Try to resolve project UUIDs from cache first, + # and then keystone + if sample.project_id: + sample.project_name = \ + self.resolve_uuid_from_cache( + "projects", sample.project_id + ) + + # Try to resolve user UUIDs from cache first, + # and then keystone + if sample.user_id: + sample.user_name = \ + self.resolve_uuid_from_cache( + "users", sample.user_id + ) + sample_dict = ( publisher_utils.meter_message_from_counter( sample, self._telemetry_secret diff --git a/ceilometer/publisher/utils.py b/ceilometer/publisher/utils.py index 75df2b7051..0d1e7be0fb 100644 --- a/ceilometer/publisher/utils.py +++ b/ceilometer/publisher/utils.py @@ -126,7 +126,9 @@ def meter_message_from_counter(sample, secret): 'counter_unit': sample.unit, 'counter_volume': sample.volume, 'user_id': sample.user_id, + 'user_name': sample.user_name, 'project_id': sample.project_id, + 'project_name': sample.project_name, 'resource_id': sample.resource_id, 'timestamp': sample.timestamp, 'resource_metadata': sample.resource_metadata, diff --git a/ceilometer/sample.py b/ceilometer/sample.py index c86caa35f9..536b561ddd 100644 --- a/ceilometer/sample.py +++ b/ceilometer/sample.py @@ -94,13 +94,16 @@ class Sample(object): def __init__(self, name, type, unit, volume, user_id, project_id, resource_id, timestamp=None, resource_metadata=None, - source=None, id=None, monotonic_time=None): + source=None, id=None, monotonic_time=None, + user_name=None, project_name=None): self.name = name self.type = type self.unit = unit self.volume = volume self.user_id = user_id + self.user_name = user_name self.project_id = project_id + self.project_name = project_name self.resource_id = resource_id self.timestamp = timestamp self.resource_metadata = resource_metadata or {} diff --git a/ceilometer/tests/unit/polling/test_manager.py b/ceilometer/tests/unit/polling/test_manager.py index e805b9be10..963e841cf8 100644 --- a/ceilometer/tests/unit/polling/test_manager.py +++ b/ceilometer/tests/unit/polling/test_manager.py @@ -378,6 +378,17 @@ class TestPollingAgent(BaseAgent): super(TestPollingAgent, self).setUp() self.mgr = self.create_manager() self.mgr.extensions = self.create_extension_list() + ks_client = mock.Mock(auth_token='fake_token') + ks_client.projects.get.return_value = mock.Mock( + name='admin', id='4465ecd1438b4d23a866cf8447387a7b' + ) + ks_client.users.get.return_value = mock.Mock( + name='admin', id='c0c935468e654d5a8baae1a08adf4dfb' + ) + self.useFixture(fixtures.MockPatch( + 'ceilometer.keystone_client.get_client', + return_value=ks_client)) + self.ks_client = ks_client self.setup_polling() @mock.patch('ceilometer.polling.manager.PollingManager') diff --git a/releasenotes/notes/add-tenant-name-discovery-668260bb4b2b0e8c.yaml b/releasenotes/notes/add-tenant-name-discovery-668260bb4b2b0e8c.yaml new file mode 100644 index 0000000000..43b6042b7d --- /dev/null +++ b/releasenotes/notes/add-tenant-name-discovery-668260bb4b2b0e8c.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Identify user and projects names with the help of their UUIDs + in the polled samples. If they are identified, set "project_name" + and "user_name" fields in the sample to the corresponding values.