From 7cba277d798c07410b9b41bef945b83e8c4a16e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Weing=C3=A4rtner?= Date: Tue, 5 Nov 2019 19:06:26 -0300 Subject: [PATCH] Dynamic pollster system to support non-OpenStack APIs The goal of this PR is to add the support for non-OpenStack APIs into Ceilometer. An example of such API is the RadosGW usage API. Change-Id: If5e1c9bce9e2709746338e043b20d328d8fb4504 --- ceilometer/declarative.py | 6 +- .../non_openstack_credentials_discovery.py | 59 ++++ ceilometer/polling/dynamic_pollster.py | 46 ++- ceilometer/polling/manager.py | 14 +- .../polling/non_openstack_dynamic_pollster.py | 144 +++++++++ ceilometer/publisher/gnocchi.py | 3 +- .../unit/polling/test_dynamic_pollster.py | 14 +- ceilometer/tests/unit/polling/test_manager.py | 25 ++ ...est_non_openstack_credentials_discovery.py | 115 ++++++++ .../test_non_openstack_dynamic_pollster.py | 273 ++++++++++++++++++ .../tests/unit/publisher/test_gnocchi.py | 2 + .../admin/telemetry-dynamic-pollster.rst | 94 +++++- ...r-non-openstack-apis-4e06694f223f34f3.yaml | 6 + setup.cfg | 1 + 14 files changed, 771 insertions(+), 31 deletions(-) create mode 100644 ceilometer/polling/discovery/non_openstack_credentials_discovery.py create mode 100644 ceilometer/polling/non_openstack_dynamic_pollster.py create mode 100644 ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py create mode 100644 ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py create mode 100644 releasenotes/notes/dynamic-pollster-system-for-non-openstack-apis-4e06694f223f34f3.yaml diff --git a/ceilometer/declarative.py b/ceilometer/declarative.py index da2c3cf13b..11480b6fe2 100644 --- a/ceilometer/declarative.py +++ b/ceilometer/declarative.py @@ -24,7 +24,7 @@ LOG = log.getLogger(__name__) class DefinitionException(Exception): - def __init__(self, message, definition_cfg): + def __init__(self, message, definition_cfg=None): msg = '%s %s: %s' % (self.__class__.__name__, definition_cfg, message) super(DefinitionException, self).__init__(msg) self.brief_message = message @@ -46,6 +46,10 @@ class DynamicPollsterDefinitionException(DefinitionException): pass +class NonOpenStackApisDynamicPollsterException(DefinitionException): + pass + + class Definition(object): JSONPATH_RW_PARSER = parser.ExtentedJsonPathParser() GETTERS_CACHE = {} diff --git a/ceilometer/polling/discovery/non_openstack_credentials_discovery.py b/ceilometer/polling/discovery/non_openstack_credentials_discovery.py new file mode 100644 index 0000000000..956a69c723 --- /dev/null +++ b/ceilometer/polling/discovery/non_openstack_credentials_discovery.py @@ -0,0 +1,59 @@ +# Copyright 2014-2015 Red Hat, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log + +from ceilometer.polling.discovery.endpoint import EndpointDiscovery + +import six.moves.urllib.parse as urlparse + +import requests + +LOG = log.getLogger(__name__) + + +class NonOpenStackCredentialsDiscovery(EndpointDiscovery): + """Barbican secrets discovery + + Discovery that supplies non-OpenStack credentials for the dynamic + pollster sub-system. This solution uses the EndpointDiscovery to + find the Barbican URL where we can retrieve the credentials. + """ + + BARBICAN_URL_GET_PAYLOAD_PATTERN = "/v1/secrets/%s/payload" + + def discover(self, manager, param=None): + barbican_secret = "No secrets found" + if not param: + return [barbican_secret] + barbican_endpoints = super(NonOpenStackCredentialsDiscovery, + self).discover("key-manager") + if not barbican_endpoints: + LOG.warning("No Barbican endpoints found to execute the" + " credentials discovery process to [%s].", + param) + return [barbican_secret] + else: + LOG.debug("Barbican endpoint found [%s].", barbican_endpoints) + + barbican_server = next(iter(barbican_endpoints)) + barbican_endpoint = self.BARBICAN_URL_GET_PAYLOAD_PATTERN % param + babrican_url = urlparse.urljoin(barbican_server, barbican_endpoint) + + LOG.debug("Retrieving secrets from: %s.", babrican_url) + resp = manager._keystone.session.get(babrican_url, authenticated=True) + if resp.status_code != requests.codes.ok: + resp.raise_for_status() + + return [resp._content] diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py index 61def7a4ad..f501c1b52c 100644 --- a/ceilometer/polling/dynamic_pollster.py +++ b/ceilometer/polling/dynamic_pollster.py @@ -40,20 +40,23 @@ class DynamicPollster(plugin_base.PollsterBase): OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values', 'value_mapping', 'default_value', 'metadata_mapping', - 'preserve_mapped_metadata' + 'preserve_mapped_metadata', 'response_entries_key'] REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit', 'value_attribute', 'endpoint_type', 'url_path'] - ALL_POLLSTER_FIELDS = OPTIONAL_POLLSTER_FIELDS + REQUIRED_POLLSTER_FIELDS - + # Mandatory name field name = "" def __init__(self, pollster_definitions, conf=None): super(DynamicPollster, self).__init__(conf) - LOG.debug("Dynamic pollster created with [%s]", + + self.ALL_POLLSTER_FIELDS =\ + self.OPTIONAL_POLLSTER_FIELDS + self.REQUIRED_POLLSTER_FIELDS + + LOG.debug("%s instantiated with [%s]", __name__, pollster_definitions) self.pollster_definitions = pollster_definitions @@ -63,9 +66,12 @@ class DynamicPollster(plugin_base.PollsterBase): LOG.debug("Metadata fields configured to [%s].", self.pollster_definitions['metadata_fields']) + self.set_default_values() + self.name = self.pollster_definitions['name'] self.obj = self + def set_default_values(self): if 'skip_sample_values' not in self.pollster_definitions: self.pollster_definitions['skip_sample_values'] = [] @@ -113,17 +119,17 @@ class DynamicPollster(plugin_base.PollsterBase): LOG.debug("No resources received for processing.") yield None - for endpoint in resources: - LOG.debug("Executing get sample on URL [%s].", endpoint) + for r in resources: + LOG.debug("Executing get sample for resource [%s].", r) samples = list([]) try: samples = self.execute_request_get_samples( - keystone_client=manager._keystone, endpoint=endpoint) + keystone_client=manager._keystone, resource=r) except RequestException as e: LOG.warning("Error [%s] while loading samples for [%s] " "for dynamic pollster [%s].", - e, endpoint, self.name) + e, r, self.name) for pollster_sample in samples: response_value_attribute_name = self.pollster_definitions[ @@ -149,6 +155,10 @@ class DynamicPollster(plugin_base.PollsterBase): if 'project_id' in pollster_sample: project_id = pollster_sample["project_id"] + resource_id = None + if 'id' in pollster_sample: + resource_id = pollster_sample["id"] + metadata = [] if 'metadata_fields' in self.pollster_definitions: metadata = dict((k, pollster_sample.get(k)) @@ -165,7 +175,7 @@ class DynamicPollster(plugin_base.PollsterBase): user_id=user_id, project_id=project_id, - resource_id=pollster_sample["id"], + resource_id=resource_id, resource_metadata=metadata ) @@ -213,12 +223,8 @@ class DynamicPollster(plugin_base.PollsterBase): def default_discovery(self): return 'endpoint:' + self.pollster_definitions['endpoint_type'] - def execute_request_get_samples(self, keystone_client, endpoint): - url = url_parse.urljoin( - endpoint, self.pollster_definitions['url_path']) - resp = keystone_client.session.get(url, authenticated=True) - if resp.status_code != requests.codes.ok: - resp.raise_for_status() + def execute_request_get_samples(self, **kwargs): + resp, url = self.internal_execute_request_get_samples(kwargs) response_json = resp.json() @@ -231,6 +237,16 @@ class DynamicPollster(plugin_base.PollsterBase): return self.retrieve_entries_from_response(response_json) return [] + def internal_execute_request_get_samples(self, kwargs): + keystone_client = kwargs['keystone_client'] + endpoint = kwargs['resource'] + url = url_parse.urljoin( + endpoint, self.pollster_definitions['url_path']) + resp = keystone_client.session.get(url, authenticated=True) + if resp.status_code != requests.codes.ok: + resp.raise_for_status() + return resp, url + def retrieve_entries_from_response(self, response_json): if isinstance(response_json, list): return response_json diff --git a/ceilometer/polling/manager.py b/ceilometer/polling/manager.py index 240e8b6abe..3329d16ed7 100644 --- a/ceilometer/polling/manager.py +++ b/ceilometer/polling/manager.py @@ -40,6 +40,7 @@ from ceilometer import declarative from ceilometer import keystone_client from ceilometer import messaging from ceilometer.polling import dynamic_pollster +from ceilometer.polling import non_openstack_dynamic_pollster from ceilometer.polling import plugin_base from ceilometer.publisher import utils as publisher_utils from ceilometer import utils @@ -338,10 +339,8 @@ class AgentManager(cotyledon.Service): LOG.info("Loading dynamic pollster [%s] from file [%s].", pollster_name, pollsters_definitions_file) try: - dynamic_pollster_object = dynamic_pollster.\ - DynamicPollster(pollster_cfg, self.conf) - pollsters_definitions[pollster_name] = \ - dynamic_pollster_object + pollsters_definitions[pollster_name] =\ + self.instantiate_dynamic_pollster(pollster_cfg) except Exception as e: LOG.error( "Error [%s] while loading dynamic pollster [%s].", @@ -356,6 +355,13 @@ class AgentManager(cotyledon.Service): len(pollsters_definitions)) return pollsters_definitions.values() + def instantiate_dynamic_pollster(self, pollster_cfg): + if 'module' in pollster_cfg: + return non_openstack_dynamic_pollster\ + .NonOpenStackApisDynamicPollster(pollster_cfg, self.conf) + else: + return dynamic_pollster.DynamicPollster(pollster_cfg, self.conf) + @staticmethod def _get_ext_mgr(namespace, *args, **kwargs): def _catch_extension_load_error(mgr, ep, exc): diff --git a/ceilometer/polling/non_openstack_dynamic_pollster.py b/ceilometer/polling/non_openstack_dynamic_pollster.py new file mode 100644 index 0000000000..2854b9216c --- /dev/null +++ b/ceilometer/polling/non_openstack_dynamic_pollster.py @@ -0,0 +1,144 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +"""Non-OpenStack Dynamic pollster component + This component enables operators to create pollsters on the fly + via configuration for non-OpenStack APIs. This appraoch is quite + useful when adding metrics from APIs such as RadosGW into the Cloud + rating and billing modules. +""" +import copy +import requests + +from ceilometer.declarative import NonOpenStackApisDynamicPollsterException +from ceilometer.polling.dynamic_pollster import DynamicPollster +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class NonOpenStackApisDynamicPollster(DynamicPollster): + + POLLSTER_REQUIRED_POLLSTER_FIELDS = ['module', 'authentication_object'] + + POLLSTER_OPTIONAL_POLLSTER_FIELDS = ['user_id_attribute', + 'project_id_attribute', + 'resource_id_attribute', + 'barbican_secret_id', + 'authentication_parameters' + ] + + def __init__(self, pollster_definitions, conf=None): + # Making sure that we do not change anything in parent classes + self.REQUIRED_POLLSTER_FIELDS = copy.deepcopy( + DynamicPollster.REQUIRED_POLLSTER_FIELDS) + self.OPTIONAL_POLLSTER_FIELDS = copy.deepcopy( + DynamicPollster.OPTIONAL_POLLSTER_FIELDS) + + # Non-OpenStack dynamic pollster do not need the 'endpoint_type'. + self.REQUIRED_POLLSTER_FIELDS.remove('endpoint_type') + + self.REQUIRED_POLLSTER_FIELDS += self.POLLSTER_REQUIRED_POLLSTER_FIELDS + self.OPTIONAL_POLLSTER_FIELDS += self.POLLSTER_OPTIONAL_POLLSTER_FIELDS + + super(NonOpenStackApisDynamicPollster, self).__init__( + pollster_definitions, conf) + + def set_default_values(self): + super(NonOpenStackApisDynamicPollster, self).set_default_values() + + if 'user_id_attribute' not in self.pollster_definitions: + self.pollster_definitions['user_id_attribute'] = None + + if 'project_id_attribute' not in self.pollster_definitions: + self.pollster_definitions['project_id_attribute'] = None + + if 'resource_id_attribute' not in self.pollster_definitions: + self.pollster_definitions['resource_id_attribute'] = None + + if 'barbican_secret_id' not in self.pollster_definitions: + self.pollster_definitions['barbican_secret_id'] = "" + + if 'authentication_parameters' not in self.pollster_definitions: + self.pollster_definitions['authentication_parameters'] = "" + + @property + def default_discovery(self): + return 'barbican:' + self.pollster_definitions['barbican_secret_id'] + + def internal_execute_request_get_samples(self, kwargs): + credentials = kwargs['resource'] + + override_credentials = self.pollster_definitions[ + 'authentication_parameters'] + if override_credentials: + credentials = override_credentials + + url = self.pollster_definitions['url_path'] + + authenticator_module_name = self.pollster_definitions['module'] + authenticator_class_name = \ + self.pollster_definitions['authentication_object'] + + imported_module = __import__(authenticator_module_name) + authenticator_class = getattr(imported_module, + authenticator_class_name) + + authenticator_arguments = list(map(str.strip, credentials.split(","))) + authenticator_instance = authenticator_class(*authenticator_arguments) + + resp = requests.get( + url, + auth=authenticator_instance) + + if resp.status_code != requests.codes.ok: + raise NonOpenStackApisDynamicPollsterException( + "Error while executing request[%s]." + " Status[%s] and reason [%s]." + % (url, resp.status_code, resp.reason)) + + return resp, url + + def execute_request_get_samples(self, **kwargs): + samples = super(NonOpenStackApisDynamicPollster, + self).execute_request_get_samples(**kwargs) + + if samples: + user_id_attribute = self.pollster_definitions[ + 'user_id_attribute'] + project_id_attribute = self.pollster_definitions[ + 'project_id_attribute'] + resource_id_attribute = self.pollster_definitions[ + 'resource_id_attribute'] + + for sample in samples: + self.generate_new_attributes_in_sample( + sample, user_id_attribute, 'user_id') + self.generate_new_attributes_in_sample( + sample, project_id_attribute, 'project_id') + self.generate_new_attributes_in_sample( + sample, resource_id_attribute, 'id') + + return samples + + def generate_new_attributes_in_sample( + self, sample, attribute_key, new_attribute_key): + if attribute_key: + attribute_value = self.retrieve_attribute_nested_value( + sample, attribute_key) + + LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].", + attribute_key, attribute_value, sample) + + sample[new_attribute_key] = attribute_value diff --git a/ceilometer/publisher/gnocchi.py b/ceilometer/publisher/gnocchi.py index 8935714160..9f51bc4d04 100644 --- a/ceilometer/publisher/gnocchi.py +++ b/ceilometer/publisher/gnocchi.py @@ -338,9 +338,10 @@ class GnocchiPublisher(publisher.ConfigPublisherBase): for resource_id, samples_of_resource in resource_grouped_samples: # NOTE(sileht): / is forbidden by Gnocchi resource_id = resource_id.replace('/', '_') - for sample in samples_of_resource: metric_name = sample.name + LOG.debug("Processing sample [%s] for resource ID [%s].", + sample, resource_id) rd = self.metric_map.get(metric_name) if rd is None: if metric_name not in self._already_logged_metric_names: diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py index 3f71adf453..3cbccf6e5d 100644 --- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py @@ -11,7 +11,7 @@ # License for the specific language governing permissions and limitations # under the License. -"""Tests for ceilometer/central/manager.py +"""Tests for ceilometer/polling/dynamic_pollster.py """ @@ -162,7 +162,8 @@ class TestDynamicPollster(base.BaseTestCase): client_mock.session.get.return_value = return_value samples = pollster.execute_request_get_samples( - client_mock, "https://endpoint.server.name/") + keystone_client=client_mock, + resource="https://endpoint.server.name/") self.assertEqual(0, len(samples)) @@ -179,7 +180,8 @@ class TestDynamicPollster(base.BaseTestCase): client_mock.session.get.return_value = return_value samples = pollster.execute_request_get_samples( - client_mock, "https://endpoint.server.name/") + keystone_client=client_mock, + resource="https://endpoint.server.name/") self.assertEqual(3, len(samples)) @@ -196,8 +198,8 @@ class TestDynamicPollster(base.BaseTestCase): exception = self.assertRaises(requests.HTTPError, pollster.execute_request_get_samples, - client_mock, - "https://endpoint.server.name/") + keystone_client=client_mock, + resource="https://endpoint.server.name/") self.assertEqual("Mock HTTP error.", str(exception)) def test_generate_new_metadata_fields_no_metadata_mapping(self): @@ -325,7 +327,7 @@ class TestDynamicPollster(base.BaseTestCase): self.assertEqual(0, len(samples_list)) - def fake_sample_list(self, keystone_client=None, endpoint=None): + def fake_sample_list(self, keystone_client=None, resource=None): samples_list = list() samples_list.append( {'name': "sample5", 'volume': 5, 'description': "desc-sample-5", diff --git a/ceilometer/tests/unit/polling/test_manager.py b/ceilometer/tests/unit/polling/test_manager.py index db96bc5ebe..3ac7af7c76 100644 --- a/ceilometer/tests/unit/polling/test_manager.py +++ b/ceilometer/tests/unit/polling/test_manager.py @@ -26,10 +26,14 @@ from stevedore import extension from ceilometer.compute import discovery as nova_discover from ceilometer.hardware import discovery +from ceilometer.polling.dynamic_pollster import DynamicPollster from ceilometer.polling import manager +from ceilometer.polling.non_openstack_dynamic_pollster import \ + NonOpenStackApisDynamicPollster from ceilometer.polling import plugin_base from ceilometer import sample from ceilometer import service + from ceilometer.tests import base @@ -890,3 +894,24 @@ class TestPollingAgentPartitioned(BaseAgent): mock.call('static_3'), mock.call('static_4'), ], any_order=True) + + def test_instantiate_dynamic_pollster_standard_pollster(self): + pollster_definition_only_required_fields = { + 'name': "test-pollster", 'sample_type': "gauge", 'unit': "test", + 'value_attribute': "volume", 'endpoint_type': "test", + 'url_path': "v1/test/endpoint/fake"} + pollster = self.mgr.instantiate_dynamic_pollster( + pollster_definition_only_required_fields) + + self.assertIsInstance(pollster, DynamicPollster) + + def test_instantiate_dynamic_pollster_non_openstack_api(self): + pollster_definition_only_required_fields = { + 'name': "test-pollster", 'sample_type': "gauge", 'unit': "test", + 'value_attribute': "volume", + 'url_path': "v1/test/endpoint/fake", 'module': "module-name", + 'authentication_object': "authentication_object"} + pollster = self.mgr.instantiate_dynamic_pollster( + pollster_definition_only_required_fields) + + self.assertIsInstance(pollster, NonOpenStackApisDynamicPollster) diff --git a/ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py b/ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py new file mode 100644 index 0000000000..33cb0dbf53 --- /dev/null +++ b/ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py @@ -0,0 +1,115 @@ +# Copyright 2014-2015 Red Hat, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import mock +import requests + +from ceilometer.polling.discovery.endpoint import EndpointDiscovery +from ceilometer.polling.discovery.non_openstack_credentials_discovery import \ + NonOpenStackCredentialsDiscovery +from oslotest import base + + +class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase): + + class FakeResponse(object): + status_code = None + json_object = None + _content = "" + + def json(self): + return self.json_object + + def raise_for_status(self): + raise requests.HTTPError("Mock HTTP error.", response=self) + + class FakeManager(object): + def __init__(self, keystone_client_mock): + self._keystone = keystone_client_mock + + def setUp(self): + super(TestNonOpenStackCredentialsDiscovery, self).setUp() + + self.discovery = NonOpenStackCredentialsDiscovery(None) + + def test_discover_no_parameters(self): + result = self.discovery.discover(None, None) + + self.assertEqual(['No secrets found'], result) + + result = self.discovery.discover(None, "") + + self.assertEqual(['No secrets found'], result) + + def test_discover_no_barbican_endpoint(self): + def discover_mock(self, type): + return [] + + original_discover_method = EndpointDiscovery.discover + EndpointDiscovery.discover = discover_mock + + result = self.discovery.discover(None, "param") + + self.assertEqual(['No secrets found'], result) + + EndpointDiscovery.discover = original_discover_method + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_discover_error_response(self, client_mock): + def discover_mock(self, type): + return ["barbican_url"] + + original_discover_method = EndpointDiscovery.discover + EndpointDiscovery.discover = discover_mock + + for http_status_code in requests.status_codes._codes.keys(): + if http_status_code < 400: + continue + + return_value = self.FakeResponse() + return_value.status_code = http_status_code + return_value.json_object = {} + + client_mock.session.get.return_value = return_value + + exception = self.assertRaises( + requests.HTTPError, + self.discovery.discover, + manager=self.FakeManager(client_mock), + param="param") + + self.assertEqual("Mock HTTP error.", str(exception)) + + EndpointDiscovery.discover = original_discover_method + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_discover_response_ok(self, client_mock): + def discover_mock(self, type): + return ["barbican_url"] + + original_discover_method = EndpointDiscovery.discover + EndpointDiscovery.discover = discover_mock + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value.json_object = {} + return_value._content = "content" + + client_mock.session.get.return_value = return_value + + response = self.discovery.discover( + manager=self.FakeManager(client_mock), param="param") + + self.assertEqual(["content"], response) + + EndpointDiscovery.discover = original_discover_method diff --git a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py new file mode 100644 index 0000000000..f0a35b647a --- /dev/null +++ b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py @@ -0,0 +1,273 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests for ceilometer/polling/non_openstack_dynamic_pollster.py +""" +import copy +import sys + +import mock +import requests + +from ceilometer.declarative import DynamicPollsterDefinitionException +from ceilometer.declarative import NonOpenStackApisDynamicPollsterException +from ceilometer.polling.dynamic_pollster import DynamicPollster +from ceilometer.polling.non_openstack_dynamic_pollster\ + import NonOpenStackApisDynamicPollster + +from oslotest import base + + +class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): + + class FakeResponse(object): + status_code = None + json_object = None + + def json(self): + return self.json_object + + def raise_for_status(self): + raise requests.HTTPError("Mock HTTP error.", response=self) + + def setUp(self): + super(TestNonOpenStackApisDynamicPollster, self).setUp() + self.pollster_definition_only_required_fields = { + 'name': "test-pollster", 'sample_type': "gauge", 'unit': "test", + 'value_attribute': "volume", + 'url_path': "v1/test/endpoint/fake", 'module': "module-name", + 'authentication_object': "authentication_object"} + + self.pollster_definition_all_fields = { + 'name': "test-pollster", 'sample_type': "gauge", 'unit': "test", + 'value_attribute': "volume", + 'url_path': "v1/test/endpoint/fake", 'module': "module-name", + 'authentication_object': "authentication_object", + 'user_id_attribute': 'user_id', + 'project_id_attribute': 'project_id', + 'resource_id_attribute': 'id', 'barbican_secret_id': 'barbican_id', + 'authentication_parameters': 'parameters'} + + def test_all_fields(self): + pollster = NonOpenStackApisDynamicPollster( + self.pollster_definition_only_required_fields) + + all_required = ['module', 'authentication_object', 'name', + 'sample_type', 'unit', 'value_attribute', + 'url_path'] + + all_optional = ['metadata_fields', 'skip_sample_values', + 'value_mapping', 'default_value', 'metadata_mapping', + 'preserve_mapped_metadata', 'user_id_attribute', + 'project_id_attribute', 'resource_id_attribute', + 'barbican_secret_id', 'authentication_parameters', + 'response_entries_key'] + all_required + + for field in all_required: + self.assertIn(field, pollster.REQUIRED_POLLSTER_FIELDS) + + for field in all_optional: + self.assertIn(field, pollster.ALL_POLLSTER_FIELDS) + + def test_all_required_fields_exceptions(self): + pollster = NonOpenStackApisDynamicPollster( + self.pollster_definition_only_required_fields) + + for key in pollster.REQUIRED_POLLSTER_FIELDS: + pollster_definition = copy.deepcopy( + self.pollster_definition_only_required_fields) + pollster_definition.pop(key) + exception = self.assertRaises(DynamicPollsterDefinitionException, + NonOpenStackApisDynamicPollster, + pollster_definition) + self.assertEqual("Required fields ['%s'] not specified." + % key, exception.brief_message) + + def test_set_default_values(self): + pollster = NonOpenStackApisDynamicPollster( + self.pollster_definition_only_required_fields) + + pollster_definitions = pollster.pollster_definitions + + self.assertEqual(None, pollster_definitions['user_id_attribute']) + self.assertEqual(None, pollster_definitions['project_id_attribute']) + self.assertEqual(None, pollster_definitions['resource_id_attribute']) + self.assertEqual('', pollster_definitions['barbican_secret_id']) + self.assertEqual('', pollster_definitions['authentication_parameters']) + + def test_user_set_optional_parameters(self): + pollster = NonOpenStackApisDynamicPollster( + self.pollster_definition_all_fields) + pollster_definitions = pollster.pollster_definitions + + self.assertEqual('user_id', + pollster_definitions['user_id_attribute']) + self.assertEqual('project_id', + pollster_definitions['project_id_attribute']) + self.assertEqual('id', + pollster_definitions['resource_id_attribute']) + self.assertEqual('barbican_id', + pollster_definitions['barbican_secret_id']) + self.assertEqual('parameters', + pollster_definitions['authentication_parameters']) + + def test_default_discovery_empty_secret_id(self): + pollster = NonOpenStackApisDynamicPollster( + self.pollster_definition_only_required_fields) + + self.assertEqual("barbican:", pollster.default_discovery) + + def test_default_discovery_not_empty_secret_id(self): + pollster = NonOpenStackApisDynamicPollster( + self.pollster_definition_all_fields) + + self.assertEqual("barbican:barbican_id", pollster.default_discovery) + + @mock.patch('requests.get') + def test_internal_execute_request_get_samples_status_code_ok( + self, get_mock): + sys.modules['module-name'] = mock.MagicMock() + + pollster = NonOpenStackApisDynamicPollster( + self.pollster_definition_only_required_fields) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value.json_object = {} + return_value.reason = "Ok" + + get_mock.return_value = return_value + + kwargs = {'resource': "credentials"} + + resp, url = pollster.internal_execute_request_get_samples(kwargs) + + self.assertEqual( + self.pollster_definition_only_required_fields['url_path'], url) + self.assertEqual(return_value, resp) + + @mock.patch('requests.get') + def test_internal_execute_request_get_samples_status_code_not_ok( + self, get_mock): + sys.modules['module-name'] = mock.MagicMock() + + pollster = NonOpenStackApisDynamicPollster( + self.pollster_definition_only_required_fields) + + for http_status_code in requests.status_codes._codes.keys(): + if http_status_code >= 400: + return_value = self.FakeResponse() + return_value.status_code = http_status_code + return_value.json_object = {} + return_value.reason = requests.status_codes._codes[ + http_status_code][0] + + get_mock.return_value = return_value + + kwargs = {'resource': "credentials"} + exception = self.assertRaises( + NonOpenStackApisDynamicPollsterException, + pollster.internal_execute_request_get_samples, kwargs) + + self.assertEqual( + "NonOpenStackApisDynamicPollsterException" + " None: Error while executing request[%s]." + " Status[%s] and reason [%s]." + % + (self.pollster_definition_only_required_fields['url_path'], + http_status_code, return_value.reason), str(exception)) + + def test_generate_new_attributes_in_sample_attribute_key_none(self): + pollster = NonOpenStackApisDynamicPollster( + self.pollster_definition_only_required_fields) + + sample = {"test": "2"} + new_key = "new-key" + + pollster.generate_new_attributes_in_sample(sample, None, new_key) + pollster.generate_new_attributes_in_sample(sample, "", new_key) + + self.assertNotIn(new_key, sample) + + def test_generate_new_attributes_in_sample(self): + pollster = NonOpenStackApisDynamicPollster( + self.pollster_definition_only_required_fields) + + sample = {"test": "2"} + new_key = "new-key" + + pollster.generate_new_attributes_in_sample(sample, "test", new_key) + + self.assertIn(new_key, sample) + self.assertEqual(sample["test"], sample[new_key]) + + def test_execute_request_get_samples_non_empty_keys(self): + sample = {'user_id_attribute': "123456789", + 'project_id_attribute': "dfghyt432345t", + 'resource_id_attribute': "sdfghjt543"} + + def execute_request_get_samples_mock(self, **kwargs): + samples = [sample] + return samples + + DynamicPollster.execute_request_get_samples =\ + execute_request_get_samples_mock + + self.pollster_definition_all_fields[ + 'user_id_attribute'] = 'user_id_attribute' + self.pollster_definition_all_fields[ + 'project_id_attribute'] = 'project_id_attribute' + self.pollster_definition_all_fields[ + 'resource_id_attribute'] = 'resource_id_attribute' + + pollster = NonOpenStackApisDynamicPollster( + self.pollster_definition_all_fields) + + params = {"d": "d"} + response = pollster.execute_request_get_samples(**params) + + self.assertEqual(sample['user_id_attribute'], + response[0]['user_id']) + self.assertEqual(sample['project_id_attribute'], + response[0]['project_id']) + self.assertEqual(sample['resource_id_attribute'], + response[0]['id']) + + def test_execute_request_get_samples_empty_keys(self): + sample = {'user_id_attribute': "123456789", + 'project_id_attribute': "dfghyt432345t", + 'resource_id_attribute': "sdfghjt543"} + + def execute_request_get_samples_mock(self, **kwargs): + samples = [sample] + return samples + + DynamicPollster.execute_request_get_samples =\ + execute_request_get_samples_mock + + self.pollster_definition_all_fields[ + 'user_id_attribute'] = None + self.pollster_definition_all_fields[ + 'project_id_attribute'] = None + self.pollster_definition_all_fields[ + 'resource_id_attribute'] = None + + pollster = NonOpenStackApisDynamicPollster( + self.pollster_definition_all_fields) + + params = {"d": "d"} + response = pollster.execute_request_get_samples(**params) + + self.assertNotIn('user_id', response[0]) + self.assertNotIn('project_id', response[0]) + self.assertNotIn('id', response[0]) diff --git a/ceilometer/tests/unit/publisher/test_gnocchi.py b/ceilometer/tests/unit/publisher/test_gnocchi.py index af83c58714..28d17fea15 100644 --- a/ceilometer/tests/unit/publisher/test_gnocchi.py +++ b/ceilometer/tests/unit/publisher/test_gnocchi.py @@ -618,6 +618,8 @@ class PublisherWorkflowTest(base.BaseTestCase, expected_debug = [ mock.call('filtered project found: %s', 'a2d42c23-d518-46b6-96ab-3fba2e146859'), + mock.call('Processing sample [%s] for resource ID [%s].', + self.sample, resource_id), ] measures_posted = False diff --git a/doc/source/admin/telemetry-dynamic-pollster.rst b/doc/source/admin/telemetry-dynamic-pollster.rst index fa4d6d8fd2..72421f78f9 100644 --- a/doc/source/admin/telemetry-dynamic-pollster.rst +++ b/doc/source/admin/telemetry-dynamic-pollster.rst @@ -24,14 +24,12 @@ dynamic pollster system: fashion. This feature is "a nice" to have, but is currently not implemented. -* non-OpenStack APIs such as RadosGW (currently in development) - * APIs that return a list of entries directly, without a first key for the list. An example is Aodh alarm list. -The dynamic pollsters system configuration ------------------------------------------- +The dynamic pollsters system configuration (for OpenStack APIs) +--------------------------------------------------------------- Each YAML file in the dynamic pollster feature can use the following attributes to define a dynamic pollster: @@ -243,3 +241,91 @@ desires): metadata_mapping: name: "display_name" default_value: 0 + +The dynamic pollsters system configuration (for non-OpenStack APIs) +------------------------------------------------------------------- + +The dynamic pollster system can also be used for non-OpenStack APIs. +to configure non-OpenStack APIs, one can use all but one attribute of +the Dynamic pollster system. The attribute that is not supported is +the ``endpoint_type``. The dynamic pollster system for non-OpenStack APIs +is activated automatically when one uses the configurations ``module``. + +The extra parameters that are available when using the Non-OpenStack +dynamic pollster sub-subsystem are the following: + +* ``module``: required parameter. It is the python module name that Ceilometer + has to load to use the authentication object when executing requests against + the API. For instance, if one wants to create a pollster to gather data from + RadosGW, he/she can use the ``awsauth`` python module. + +* ``authentication_object``: mandatory parameter. The name of the class that we + can find in the ``module`` that Ceilometer will use as the authentication + object in the request. For instance, when using the ``awsauth`` python module + to gather data from RadosGW, one can use the authentication object as + ``S3Auth``. + +* ``authentication_parameters``: optional parameter. It is a comma separated + value that will be used to instantiate the ``authentication_object``. For + instance, if we gather data from RadosGW, and we use the ``S3Auth`` class, + the ``authentication_parameters`` can be configured as + ``, rados_gw_secret_key, rados_gw_host_name``. + +* ``barbican_secret_id``: optional parameter. The Barbican secret ID, + from which, Ceilometer can retrieve the comma separated values of the + ``authentication_parameters``. + +* ``user_id_attribute``: optional parameter. The name of the attribute in the + entries that are processed from ``response_entries_key`` elements that + will be mapped to ``user_id`` attribute that is sent to Gnocchi. + +* ``project_id_attribute``: optional parameter. The name of the attribute in + the entries that are processed from ``response_entries_key`` elements that + will be mapped to ``project_id`` attribute that is sent to Gnocchi. + +* ``resource_id_attribute``: optional parameter. The name of the attribute + in the entries that are processed from ``response_entries_key`` elements that + will be mapped to ``id`` attribute that is sent to Gnocchi. + +As follows we present an example on how to convert the hard-coded pollster +for `radosgw.api.request` metric to the dynamic pollster model: + +.. code-block:: yaml + + --- + + - name: "dynamic.radosgw.api.request" + sample_type: "gauge" + unit: "request" + value_attribute: "total.ops" + url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage" + module: "awsauth" + authentication_object: "S3Auth" + authentication_parameters: ", , + " + user_id_attribute: "user" + project_id_attribute: "user" + resource_id_attribute: "user" + response_entries_key: "summary" + +We can take that example a bit further, and instead of gathering the `total +.ops` variable, which counts for all the requests (even the unsuccessful +ones), we can use the `successful_ops`. + +.. code-block:: yaml + + --- + + - name: "dynamic.radosgw.api.request.successful_ops" + sample_type: "gauge" + unit: "request" + value_attribute: "total.successful_ops" + url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage" + module: "awsauth" + authentication_object: "S3Auth" + authentication_parameters: ", , + " + user_id_attribute: "user" + project_id_attribute: "user" + resource_id_attribute: "user" + response_entries_key: "summary" diff --git a/releasenotes/notes/dynamic-pollster-system-for-non-openstack-apis-4e06694f223f34f3.yaml b/releasenotes/notes/dynamic-pollster-system-for-non-openstack-apis-4e06694f223f34f3.yaml new file mode 100644 index 0000000000..69eef28f48 --- /dev/null +++ b/releasenotes/notes/dynamic-pollster-system-for-non-openstack-apis-4e06694f223f34f3.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Add the support for non-OpenStack APIs in the dynamic pollster system. + This extension enables operators to create pollster on the fly to handle + metrics from systems such as the RadosGW API. diff --git a/setup.cfg b/setup.cfg index fba735beb1..205739ba64 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,6 +50,7 @@ ceilometer.discover.compute = local_instances = ceilometer.compute.discovery:InstanceDiscovery ceilometer.discover.central = + barbican = ceilometer.polling.discovery.non_openstack_credentials_discovery:NonOpenStackCredentialsDiscovery endpoint = ceilometer.polling.discovery.endpoint:EndpointDiscovery tenant = ceilometer.polling.discovery.tenant:TenantDiscovery lb_pools = ceilometer.network.services.discovery:LBPoolsDiscovery