From 7bff46921e6a5f9c8ecae97aa3756d8c570f23c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Weing=C3=A4rtner?= Date: Thu, 15 Aug 2019 13:58:29 -0300 Subject: [PATCH] Create dynamic pollster feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dynamic pollster feature allows system administrators to create/update pollsters on the fly (without changing code). The system reads YAML configures that are found in ``pollsters_definitions_dirs``, which has the default at ``/etc/ceilometer/pollsters.d``. Each YAML file in the dynamic pollster feature can use the following attributes to define a dynamic pollster: * ``name`` -- mandatory field. It specifies the name/key of the dynamic pollster. For instance, a pollster for magnum can use the name ``dynamic.magnum.cluster``; * ``sample_type``: mandatory field; it defines the sample type. It must be one of the values: ``gauge``, ``delta``, ``cumulative``; * ``unit``: mandatory field; defines the unit of the metric that is being collected. For magnum, for instance, one can use ``cluster`` as the unit or some other meaningful String value; * ``value_attribute``: mandatory attribute; defines the attribute in the JSON response from the URL of the component being polled. In our magnum example, we can use ``status`` as the value attribute; * ``endpoint_type``: mandatory field; defines the endpoint type that is used to discover the base URL of the component to be monitored; for magnum, one can use ``container-infra``. Other values are accepted such as ``volume`` for cinder endpoints, ``object-store`` for swift, and so on; * ``url_path``: mandatory attribute. It defines the path of the request that we execute on the endpoint to gather data. For example, to gather data from magnum, one can use ``v1/clusters/detail``; * ``metadata_fields``: optional field. It is a list of all fields that the response of the request executed with ``url_path`` that we want to retrieve. As an example, for magnum, one can use the following values: ``` metadata_fields: - "labels" - "updated_at" - "keypair" - "master_flavor_id" - "api_address" - "master_addresses" - "node_count" - "docker_volume_size" - "master_count" - "node_addresses" - "status_reason" - "coe_version" - "cluster_template_id" - "name" - "stack_id" - "created_at" - "discovery_url" - "container_version" ``` * ``skip_sample_values``: optional field. It defines the values that might come in the ``value_attribute`` that we want to ignore. For magnun, one could for instance, ignore some of the status it has for clusters. Therefore, data is not gathered for clusters in the defined status. ``` skip_sample_values: - "CREATE_FAILED" - "DELETE_FAILED" ``` * ``value_mapping``: optional attribute. It defines a mapping for the values that the dynamic pollster is handling. This is the actual value that is sent to Gnocchi or other backends. If there is no mapping specified, we will use the raw value that is obtained with the use of ``value_attribute``. An example for magnum, one can use: ``` value_mapping: CREATE_IN_PROGRESS: "0" CREATE_FAILED: "1" CREATE_COMPLETE: "2" UPDATE_IN_PROGRESS: "3" UPDATE_FAILED: "4" UPDATE_COMPLETE: "5" DELETE_IN_PROGRESS: "6" DELETE_FAILED: "7" DELETE_COMPLETE: "8" RESUME_COMPLETE: "9" RESUME_FAILED: "10" RESTORE_COMPLETE: "11" ROLLBACK_IN_PROGRESS: "12" ROLLBACK_FAILED: "13" ROLLBACK_COMPLETE: "14" SNAPSHOT_COMPLETE: "15" CHECK_COMPLETE: "16" ADOPT_COMPLETE: "17" ``` * ``default_value_mapping``: optional parameter. The default value for the value mapping in case the variable value receives data that is not mapped to something in the ``value_mapping`` configuration. This attribute is only used when ``value_mapping`` is defined. Moreover, it has a default of ``-1``. Change-Id: I5f0614518a9e304b86b74aa5bb0f9667d2a3a787 Signed-off-by: Rafael Weingärtner --- .gitignore | 6 + ceilometer/cmd/polling.py | 2 +- ceilometer/declarative.py | 4 + ceilometer/opts.py | 1 + ceilometer/polling/dynamic_pollster.py | 231 +++++++++++ ceilometer/polling/manager.py | 87 +++- .../unit/polling/test_dynamic_pollster.py | 377 ++++++++++++++++++ doc/source/admin/index.rst | 1 + .../admin/telemetry-data-collection.rst | 5 + .../admin/telemetry-dynamic-pollster.rst | 231 +++++++++++ ...amic-pollster-system-6b45c8c973201b2b.yaml | 5 + 11 files changed, 943 insertions(+), 7 deletions(-) create mode 100644 ceilometer/polling/dynamic_pollster.py create mode 100644 ceilometer/tests/unit/polling/test_dynamic_pollster.py create mode 100644 doc/source/admin/telemetry-dynamic-pollster.rst create mode 100644 releasenotes/notes/dynamic-pollster-system-6b45c8c973201b2b.yaml diff --git a/.gitignore b/.gitignore index 80f766d8c9..d5ccb911dc 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,9 @@ releasenotes/build #IntelJ Idea .idea/ + +#venv +venv/ + +#Pyenv files +.python-version diff --git a/ceilometer/cmd/polling.py b/ceilometer/cmd/polling.py index a1353294c6..2bc5de0573 100644 --- a/ceilometer/cmd/polling.py +++ b/ceilometer/cmd/polling.py @@ -69,7 +69,7 @@ CLI_OPTS = [ default=['compute', 'central'], dest='polling_namespaces', help='Polling namespace(s) to be used while ' - 'resource polling'), + 'resource polling') ] diff --git a/ceilometer/declarative.py b/ceilometer/declarative.py index 259aa4431b..da2c3cf13b 100644 --- a/ceilometer/declarative.py +++ b/ceilometer/declarative.py @@ -42,6 +42,10 @@ class ResourceDefinitionException(DefinitionException): pass +class DynamicPollsterDefinitionException(DefinitionException): + pass + + class Definition(object): JSONPATH_RW_PARSER = parser.ExtentedJsonPathParser() GETTERS_CACHE = {} diff --git a/ceilometer/opts.py b/ceilometer/opts.py index 8b844cc00a..196c560c5c 100644 --- a/ceilometer/opts.py +++ b/ceilometer/opts.py @@ -73,6 +73,7 @@ def list_opts(): ceilometer.compute.virt.libvirt.utils.OPTS, ceilometer.objectstore.swift.OPTS, ceilometer.pipeline.base.OPTS, + ceilometer.polling.manager.POLLING_OPTS, ceilometer.sample.OPTS, ceilometer.utils.OPTS, OPTS)), diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py new file mode 100644 index 0000000000..c2066ac3f4 --- /dev/null +++ b/ceilometer/polling/dynamic_pollster.py @@ -0,0 +1,231 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Dynamic pollster component + This component enables operators to create new pollsters on the fly + via configuration. The configuration files are read from + '/etc/ceilometer/pollsters.d/'. The pollster are defined in YAML files + similar to the idea used for handling notifications. +""" + +from oslo_log import log +from oslo_utils import timeutils +from requests import RequestException + +from ceilometer import declarative +from ceilometer.polling import plugin_base +from ceilometer import sample + + +import requests +from six.moves.urllib import parse as url_parse + +LOG = log.getLogger(__name__) + + +class DynamicPollster(plugin_base.PollsterBase): + + OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values', + 'value_mapping', 'default_value', + 'metadata_mapping', + 'preserve_mapped_metadata'] + + REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit', + 'value_attribute', 'endpoint_type', + 'url_path'] + + ALL_POLLSTER_FIELDS = OPTIONAL_POLLSTER_FIELDS + REQUIRED_POLLSTER_FIELDS + + name = "" + + def __init__(self, pollster_definitions, conf=None): + super(DynamicPollster, self).__init__(conf) + LOG.debug("Dynamic pollster created with [%s]", + pollster_definitions) + + self.pollster_definitions = pollster_definitions + self.validate_pollster_definition() + + if 'metadata_fields' in self.pollster_definitions: + LOG.debug("Metadata fields configured to [%s].", + self.pollster_definitions['metadata_fields']) + + self.name = self.pollster_definitions['name'] + self.obj = self + + if 'skip_sample_values' not in self.pollster_definitions: + self.pollster_definitions['skip_sample_values'] = [] + + if 'value_mapping' not in self.pollster_definitions: + self.pollster_definitions['value_mapping'] = {} + + if 'default_value' not in self.pollster_definitions: + self.pollster_definitions['default_value'] = -1 + + if 'preserve_mapped_metadata' not in self.pollster_definitions: + self.pollster_definitions['preserve_mapped_metadata'] = True + + if 'metadata_mapping' not in self.pollster_definitions: + self.pollster_definitions['metadata_mapping'] = {} + + def validate_pollster_definition(self): + missing_required_fields = \ + [field for field in self.REQUIRED_POLLSTER_FIELDS + if field not in self.pollster_definitions] + + if missing_required_fields: + raise declarative.DynamicPollsterDefinitionException( + "Required fields %s not specified." + % missing_required_fields, self.pollster_definitions) + + sample_type = self.pollster_definitions['sample_type'] + if sample_type not in sample.TYPES: + raise declarative.DynamicPollsterDefinitionException( + "Invalid sample type [%s]. Valid ones are [%s]." + % (sample_type, sample.TYPES), self.pollster_definitions) + + for definition_key in self.pollster_definitions: + if definition_key not in self.ALL_POLLSTER_FIELDS: + LOG.warning( + "Field [%s] defined in [%s] is unknown " + "and will be ignored. Valid fields are [%s].", + definition_key, self.pollster_definitions, + self.ALL_POLLSTER_FIELDS) + + def get_samples(self, manager, cache, resources): + if not resources: + LOG.debug("No resources received for processing.") + yield None + + for endpoint in resources: + LOG.debug("Executing get sample on URL [%s].", endpoint) + + samples = list([]) + try: + samples = self.execute_request_get_samples( + keystone_client=manager._keystone, endpoint=endpoint) + except RequestException as e: + LOG.warning("Error [%s] while loading samples for [%s] " + "for dynamic pollster [%s].", + e, endpoint, self.name) + + for pollster_sample in samples: + response_value_attribute_name = self.pollster_definitions[ + 'value_attribute'] + value = pollster_sample[response_value_attribute_name] + + skip_sample_values = \ + self.pollster_definitions['skip_sample_values'] + if skip_sample_values and value in skip_sample_values: + LOG.debug("Skipping sample [%s] because value [%s] " + "is configured to be skipped in skip list [%s].", + pollster_sample, value, skip_sample_values) + continue + + value = self.execute_value_mapping(value) + + user_id = None + if 'user_id' in pollster_sample: + user_id = pollster_sample["user_id"] + + project_id = None + if 'project_id' in pollster_sample: + project_id = pollster_sample["project_id"] + + metadata = [] + if 'metadata_fields' in self.pollster_definitions: + metadata = dict((k, pollster_sample.get(k)) + for k in self.pollster_definitions[ + 'metadata_fields']) + self.generate_new_metadata_fields(metadata=metadata) + yield sample.Sample( + timestamp=timeutils.isotime(), + + name=self.pollster_definitions['name'], + type=self.pollster_definitions['sample_type'], + unit=self.pollster_definitions['unit'], + volume=value, + + user_id=user_id, + project_id=project_id, + resource_id=pollster_sample["id"], + + resource_metadata=metadata + ) + + def execute_value_mapping(self, value): + value_mapping = self.pollster_definitions['value_mapping'] + if value_mapping: + if value in value_mapping: + old_value = value + value = value_mapping[value] + LOG.debug("Value mapped from [%s] to [%s]", + old_value, value) + else: + default_value = \ + self.pollster_definitions['default_value'] + LOG.warning( + "Value [%s] was not found in value_mapping [%s]; " + "therefore, we will use the default [%s].", + value, value_mapping, default_value) + value = default_value + return value + + def generate_new_metadata_fields(self, metadata=None): + metadata_mapping = self.pollster_definitions['metadata_mapping'] + if not metadata_mapping or not metadata: + return + + metadata_keys = list(metadata.keys()) + for k in metadata_keys: + if k not in metadata_mapping: + continue + + new_key = metadata_mapping[k] + metadata[new_key] = metadata[k] + LOG.debug("Generating new key [%s] with content [%s] of key [%s]", + new_key, metadata[k], k) + if self.pollster_definitions['preserve_mapped_metadata']: + continue + + k_value = metadata.pop(k) + LOG.debug("Removed key [%s] with value [%s] from " + "metadata set that is sent to Gnocchi.", k, k_value) + + @property + def default_discovery(self): + return 'endpoint:' + self.pollster_definitions['endpoint_type'] + + def execute_request_get_samples(self, keystone_client, endpoint): + url = url_parse.urljoin( + endpoint, self.pollster_definitions['url_path']) + resp = keystone_client.session.get(url, authenticated=True) + if resp.status_code != requests.codes.ok: + resp.raise_for_status() + + response_json = resp.json() + + entry_size = len(response_json) + LOG.debug("Entries [%s] in the JSON for request [%s] " + "for dynamic pollster [%s].", + response_json, url, self.name) + + if entry_size > 0: + first_entry_name = None + try: + first_entry_name = next(iter(response_json)) + except RuntimeError as e: + LOG.debug("Generator threw a StopIteration " + "and we need to catch it [%s].", e) + return response_json[first_entry_name] + return [] diff --git a/ceilometer/polling/manager.py b/ceilometer/polling/manager.py index dfeb3ba250..240e8b6abe 100644 --- a/ceilometer/polling/manager.py +++ b/ceilometer/polling/manager.py @@ -15,8 +15,10 @@ # under the License. import collections +import glob import itertools import logging +import os import random import uuid @@ -34,8 +36,10 @@ from stevedore import extension from tooz import coordination from ceilometer import agent +from ceilometer import declarative from ceilometer import keystone_client from ceilometer import messaging +from ceilometer.polling import dynamic_pollster from ceilometer.polling import plugin_base from ceilometer.publisher import utils as publisher_utils from ceilometer import utils @@ -58,6 +62,10 @@ POLLING_OPTS = [ default=50, help='Batch size of samples to send to notification agent, ' 'Set to 0 to disable'), + cfg.MultiStrOpt('pollsters_definitions_dirs', + default=["/etc/ceilometer/pollsters.d"], + help="List of directories with YAML files used " + "to created pollsters.") ] @@ -93,7 +101,7 @@ class Resources(object): not self.agent_manager.partition_coordinator or self.agent_manager.hashrings[ static_resources_group].belongs_to_self( - six.text_type(v))] + source_discovery + six.text_type(v))] + source_discovery return source_discovery @@ -245,8 +253,12 @@ class AgentManager(cotyledon.Service): extensions_fb = (self._extensions_from_builder('poll', namespace) for namespace in namespaces) + # Create dynamic pollsters + extensions_dynamic_pollsters = self.create_dynamic_pollsters() + self.extensions = list(itertools.chain(*list(extensions))) + list( - itertools.chain(*list(extensions_fb))) + itertools.chain(*list(extensions_fb))) + list( + extensions_dynamic_pollsters) if not self.extensions: LOG.warning('No valid pollsters can be loaded from %s ' @@ -280,6 +292,70 @@ class AgentManager(cotyledon.Service): self._keystone = None self._keystone_last_exception = None + def create_dynamic_pollsters(self): + """Creates dynamic pollsters + + This method Creates dynamic pollsters based on configurations placed on + 'pollsters_definitions_dirs' + + :return: a list with the dynamic pollsters defined by the operator. + """ + + pollsters_definitions_dirs = self.conf.pollsters_definitions_dirs + if not pollsters_definitions_dirs: + LOG.info("Variable 'pollsters_definitions_dirs' not defined.") + return [] + + LOG.info("Looking for dynamic pollsters configurations at [%s].", + pollsters_definitions_dirs) + pollsters_definitions_files = [] + for directory in pollsters_definitions_dirs: + files = glob.glob(os.path.join(directory, "*.yaml")) + if not files: + LOG.info("No dynamic pollsters found in folder [%s].", + directory) + continue + for filepath in sorted(files): + if filepath is not None: + pollsters_definitions_files.append(filepath) + + if not pollsters_definitions_files: + LOG.info("No dynamic pollsters file found in dirs [%s].", + pollsters_definitions_dirs) + return [] + + pollsters_definitions = {} + for pollsters_definitions_file in pollsters_definitions_files: + pollsters_cfg = declarative.load_definitions( + self.conf, {}, pollsters_definitions_file) + + LOG.info("File [%s] has [%s] dynamic pollster configurations.", + pollsters_definitions_file, len(pollsters_cfg)) + + for pollster_cfg in pollsters_cfg: + pollster_name = pollster_cfg['name'] + if pollster_name not in pollsters_definitions: + LOG.info("Loading dynamic pollster [%s] from file [%s].", + pollster_name, pollsters_definitions_file) + try: + dynamic_pollster_object = dynamic_pollster.\ + DynamicPollster(pollster_cfg, self.conf) + pollsters_definitions[pollster_name] = \ + dynamic_pollster_object + except Exception as e: + LOG.error( + "Error [%s] while loading dynamic pollster [%s].", + e, pollster_name) + + else: + LOG.info( + "Dynamic pollster [%s] is already defined." + "Therefore, we are skipping it.", pollster_name) + + LOG.debug("Total of dynamic pollsters [%s] loaded.", + len(pollsters_definitions)) + return pollsters_definitions.values() + @staticmethod def _get_ext_mgr(namespace, *args, **kwargs): def _catch_extension_load_error(mgr, ep, exc): @@ -371,7 +447,6 @@ class AgentManager(cotyledon.Service): futures.ThreadPoolExecutor(max_workers=len(data))) for interval, polling_task in data.items(): - @periodics.periodic(spacing=interval, run_immediately=True) def task(running_task): self.interval_task(running_task) @@ -461,9 +536,9 @@ class AgentManager(cotyledon.Service): service_type = getattr( self.conf.service_types, discoverer.KEYSTONE_REQUIRED_FOR_SERVICE) - if not keystone_client.get_service_catalog( - self.keystone).get_endpoints( - service_type=service_type): + if not keystone_client.\ + get_service_catalog(self.keystone).\ + get_endpoints(service_type=service_type): LOG.warning( 'Skipping %(name)s, %(service_type)s service ' 'is not registered in keystone', diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py new file mode 100644 index 0000000000..592fac1686 --- /dev/null +++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py @@ -0,0 +1,377 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests for ceilometer/central/manager.py +""" + + +from ceilometer.declarative import DynamicPollsterDefinitionException +from ceilometer.polling import dynamic_pollster +from ceilometer import sample + +import copy +import logging +import mock + +from oslotest import base + +import requests + +LOG = logging.getLogger(__name__) + + +class TestDynamicPollster(base.BaseTestCase): + class FakeResponse(object): + status_code = None + json_object = None + + def json(self): + return self.json_object + + def raise_for_status(self): + raise requests.HTTPError("Mock HTTP error.", response=self) + + class FakeManager(object): + _keystone = None + + def setUp(self): + super(TestDynamicPollster, self).setUp() + self.pollster_definition_only_required_fields = { + 'name': "test-pollster", 'sample_type': "gauge", 'unit': "test", + 'value_attribute': "volume", 'endpoint_type': "test", + 'url_path': "v1/test/endpoint/fake"} + + self.pollster_definition_all_fields = { + 'metadata_fields': "metadata-field-name", + 'skip_sample_values': ["I-do-not-want-entries-with-this-value"], + 'value_mapping': { + 'value-to-map': 'new-value', 'value-to-map-to-numeric': 12 + }, + 'default_value_mapping': 0, + 'metadata_mapping': { + 'old-metadata-name': "new-metadata-name" + }, + 'preserve_mapped_metadata': False} + self.pollster_definition_all_fields.update( + self.pollster_definition_only_required_fields) + + def execute_basic_asserts(self, pollster, pollster_definition): + self.assertEqual(pollster, pollster.obj) + self.assertEqual(pollster_definition['name'], pollster.name) + + for key in pollster.REQUIRED_POLLSTER_FIELDS: + self.assertEqual(pollster_definition[key], + pollster.pollster_definitions[key]) + + self.assertEqual(pollster_definition, pollster.pollster_definitions) + + def test_all_required_fields_ok(self): + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + self.execute_basic_asserts( + pollster, self.pollster_definition_only_required_fields) + + self.assertEqual( + 0, len(pollster.pollster_definitions['skip_sample_values'])) + self.assertEqual( + 0, len(pollster.pollster_definitions['value_mapping'])) + self.assertEqual( + -1, pollster.pollster_definitions['default_value']) + self.assertEqual( + 0, len(pollster.pollster_definitions['metadata_mapping'])) + self.assertEqual( + True, pollster.pollster_definitions['preserve_mapped_metadata']) + + def test_all_fields_ok(self): + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_all_fields) + + self.execute_basic_asserts(pollster, + self.pollster_definition_all_fields) + + self.assertEqual( + 1, len(pollster.pollster_definitions['skip_sample_values'])) + self.assertEqual( + 2, len(pollster.pollster_definitions['value_mapping'])) + self.assertEqual( + 0, pollster.pollster_definitions['default_value_mapping']) + self.assertEqual( + 1, len(pollster.pollster_definitions['metadata_mapping'])) + self.assertEqual( + False, pollster.pollster_definitions['preserve_mapped_metadata']) + + def test_all_required_fields_exceptions(self): + for key in dynamic_pollster.\ + DynamicPollster.REQUIRED_POLLSTER_FIELDS: + pollster_definition = copy.deepcopy( + self.pollster_definition_only_required_fields) + pollster_definition.pop(key) + exception = self.assertRaises(DynamicPollsterDefinitionException, + dynamic_pollster.DynamicPollster, + pollster_definition) + self.assertEqual("Required fields ['%s'] not specified." + % key, exception.brief_message) + + def test_invalid_sample_type(self): + self.pollster_definition_only_required_fields[ + 'sample_type'] = "invalid_sample_type" + exception = self.assertRaises( + DynamicPollsterDefinitionException, + dynamic_pollster.DynamicPollster, + self.pollster_definition_only_required_fields) + self.assertEqual("Invalid sample type [invalid_sample_type]. " + "Valid ones are [('gauge', 'delta', 'cumulative')].", + exception.brief_message) + + def test_all_valid_sample_type(self): + for sample_type in sample.TYPES: + self.pollster_definition_only_required_fields[ + 'sample_type'] = sample_type + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + self.execute_basic_asserts( + pollster, self.pollster_definition_only_required_fields) + + def test_default_discovery_method(self): + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + self.assertEqual("endpoint:test", pollster.default_discovery) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_get_samples_empty_response(self, client_mock): + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value.json_object = {} + + client_mock.session.get.return_value = return_value + + samples = pollster.execute_request_get_samples( + client_mock, "https://endpoint.server.name/") + + self.assertEqual(0, len(samples)) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_get_samples_response_non_empty( + self, client_mock): + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.ok + return_value.json_object = {"firstElement": [{}, {}, {}]} + + client_mock.session.get.return_value = return_value + + samples = pollster.execute_request_get_samples( + client_mock, "https://endpoint.server.name/") + + self.assertEqual(3, len(samples)) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_execute_request_get_samples_exception_on_request( + self, client_mock): + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + return_value = self.FakeResponse() + return_value.status_code = requests.codes.bad + + client_mock.session.get.return_value = return_value + + exception = self.assertRaises(requests.HTTPError, + pollster.execute_request_get_samples, + client_mock, + "https://endpoint.server.name/") + self.assertEqual("Mock HTTP error.", str(exception)) + + def test_generate_new_metadata_fields_no_metadata_mapping(self): + metadata = {'name': 'someName', + 'value': 1} + + metadata_before_call = copy.deepcopy(metadata) + + self.pollster_definition_only_required_fields['metadata_mapping'] = {} + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + pollster.generate_new_metadata_fields(metadata) + + self.assertEqual(metadata_before_call, metadata) + + def test_generate_new_metadata_fields_preserve_old_key(self): + metadata = {'name': 'someName', 'value': 2} + + expected_metadata = copy.deepcopy(metadata) + expected_metadata['balance'] = metadata['value'] + + self.pollster_definition_only_required_fields[ + 'metadata_mapping'] = {'value': 'balance'} + self.pollster_definition_only_required_fields[ + 'preserve_mapped_metadata'] = True + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + pollster.generate_new_metadata_fields(metadata) + + self.assertEqual(expected_metadata, metadata) + + def test_generate_new_metadata_fields_preserve_old_key_equals_false(self): + metadata = {'name': 'someName', 'value': 1} + + expected_clean_metadata = copy.deepcopy(metadata) + expected_clean_metadata['balance'] = metadata['value'] + expected_clean_metadata.pop('value') + + self.pollster_definition_only_required_fields[ + 'metadata_mapping'] = {'value': 'balance'} + self.pollster_definition_only_required_fields[ + 'preserve_mapped_metadata'] = False + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + pollster.generate_new_metadata_fields(metadata) + + self.assertEqual(expected_clean_metadata, metadata) + + def test_execute_value_mapping_no_value_mapping(self): + self.pollster_definition_only_required_fields['value_mapping'] = {} + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + value_to_be_mapped = "test" + expected_value = value_to_be_mapped + value = pollster.execute_value_mapping(value_to_be_mapped) + + self.assertEqual(expected_value, value) + + def test_execute_value_mapping_no_value_mapping_found_with_default(self): + self.pollster_definition_only_required_fields[ + 'value_mapping'] = {'some-possible-value': 15} + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + value_to_be_mapped = "test" + expected_value = -1 + value = pollster.execute_value_mapping(value_to_be_mapped) + + self.assertEqual(expected_value, value) + + def test_execute_value_mapping_no_value_mapping_found_with_custom_default( + self): + self.pollster_definition_only_required_fields[ + 'value_mapping'] = {'some-possible-value': 5} + self.pollster_definition_only_required_fields[ + 'default_value'] = 0 + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + value_to_be_mapped = "test" + expected_value = 0 + value = pollster.execute_value_mapping(value_to_be_mapped) + + self.assertEqual(expected_value, value) + + def test_execute_value_mapping(self): + self.pollster_definition_only_required_fields[ + 'value_mapping'] = {'test': 'new-value'} + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + value_to_be_mapped = "test" + expected_value = 'new-value' + value = pollster.execute_value_mapping(value_to_be_mapped) + + self.assertEqual(expected_value, value) + + def test_get_samples_no_resources(self): + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + samples = pollster.get_samples(None, None, None) + + self.assertEqual(None, next(samples)) + + @mock.patch('ceilometer.polling.dynamic_pollster.' + 'DynamicPollster.execute_request_get_samples') + def test_get_samples_empty_samples(self, execute_request_get_samples_mock): + execute_request_get_samples_mock.side_effect = [] + + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + fake_manager = self.FakeManager() + samples = pollster.get_samples( + fake_manager, None, ["https://endpoint.server.name.com/"]) + + samples_list = list() + try: + for s in samples: + samples_list.append(s) + except RuntimeError as e: + LOG.debug("Generator threw a StopIteration " + "and we need to catch it [%s]." % e) + + self.assertEqual(0, len(samples_list)) + + def fake_sample_list(self, keystone_client=None, endpoint=None): + samples_list = list() + samples_list.append( + {'name': "sample5", 'volume': 5, 'description': "desc-sample-5", + 'user_id': "924d1f77-5d75-4b96-a755-1774d6be17af", + 'project_id': "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e", + 'id': "e335c317-dfdd-4f22-809a-625bd9a5992d" + } + ) + samples_list.append( + {'name': "sample1", 'volume': 2, 'description': "desc-sample-2", + 'user_id': "20b5a704-b481-4603-a99e-2636c144b876", + 'project_id': "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e", + 'id': "2e350554-6c05-4fda-8109-e47b595a714c" + } + ) + return samples_list + + @mock.patch.object( + dynamic_pollster.DynamicPollster, + 'execute_request_get_samples', + fake_sample_list) + def test_get_samples(self): + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_only_required_fields) + + fake_manager = self.FakeManager() + samples = pollster.get_samples( + fake_manager, None, ["https://endpoint.server.name.com/"]) + + samples_list = list(samples) + self.assertEqual(2, len(samples_list)) + + first_element = [ + s for s in samples_list + if s.resource_id == "e335c317-dfdd-4f22-809a-625bd9a5992d"][0] + self.assertEqual(5, first_element.volume) + self.assertEqual( + "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e", first_element.project_id) + self.assertEqual( + "924d1f77-5d75-4b96-a755-1774d6be17af", first_element.user_id) + + second_element = [ + s for s in samples_list + if s.resource_id == "2e350554-6c05-4fda-8109-e47b595a714c"][0] + self.assertEqual(2, second_element.volume) + self.assertEqual( + "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e", second_element.project_id) + self.assertEqual( + "20b5a704-b481-4603-a99e-2636c144b876", second_element.user_id) diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst index c9ce62cc22..920d30f592 100644 --- a/doc/source/admin/index.rst +++ b/doc/source/admin/index.rst @@ -20,6 +20,7 @@ Configuration telemetry-data-collection telemetry-data-pipelines telemetry-best-practices + telemetry-dynamic-pollster Data Types ========== diff --git a/doc/source/admin/telemetry-data-collection.rst b/doc/source/admin/telemetry-data-collection.rst index 26c163536f..618c8f1f55 100644 --- a/doc/source/admin/telemetry-data-collection.rst +++ b/doc/source/admin/telemetry-data-collection.rst @@ -294,6 +294,11 @@ Some of the services polled with this agent are: To install and configure this service use the :ref:`install_rdo` section in the Installation Tutorials and Guides. +Although Ceilometer has a set of default polling agents, operators can +add new pollsters dynamically via the dynamic pollsters subsystem +:ref:`telemetry_dynamic_pollster`. + + .. _telemetry-ipmi-agent: IPMI agent diff --git a/doc/source/admin/telemetry-dynamic-pollster.rst b/doc/source/admin/telemetry-dynamic-pollster.rst new file mode 100644 index 0000000000..6fa4950ec3 --- /dev/null +++ b/doc/source/admin/telemetry-dynamic-pollster.rst @@ -0,0 +1,231 @@ +.. _telemetry_dynamic_pollster: + +Introduction to dynamic pollster subsystem +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The dynamic pollster feature allows system administrators to +create/update REST API pollsters on the fly (without changing code). +The system reads YAML configures that are found in +``pollsters_definitions_dirs`` parameter, which has the default at +``/etc/ceilometer/pollsters.d``. Operators can use a single file per +dynamic pollster or multiple dynamic pollsters per file. + + +Current limitations of the dynamic pollster system +-------------------------------------------------- +Currently, the following types of APIs are not supported by the +dynamic pollster system: + +* Paging APIs: if a user configures a dynamic pollster to gather data + from a paging API, the pollster will use only the entries from the first + page. + +* Tenant APIs: Tenant APIs are the ones that need to be polled in a tenant + fashion. This feature is "a nice" to have, but is currently not + implemented. + +* non-OpenStack APIs such as RadosGW (currently in development) + +* APIs that return a list of entries directly, without a first key for the + list. An example is Aodh alarm list. + + +The dynamic pollsters system configuration +------------------------------------------ +Each YAML file in the dynamic pollster feature can use the following +attributes to define a dynamic pollster: + +* ``name``: mandatory field. It specifies the name/key of the dynamic + pollster. For instance, a pollster for magnum can use the name + ``dynamic.magnum.cluster``; + +* ``sample_type``: mandatory field; it defines the sample type. It must + be one of the values: ``gauge``, ``delta``, ``cumulative``; + +* ``unit``: mandatory field; defines the unit of the metric that is + being collected. For magnum, for instance, one can use ``cluster`` as + the unit or some other meaningful String value; + +* ``value_attribute``: mandatory attribute; defines the attribute in the + JSON response from the URL of the component being polled. In our magnum + example, we can use ``status`` as the value attribute; + +* ``endpoint_type``: mandatory field; defines the endpoint type that is + used to discover the base URL of the component to be monitored; for + magnum, one can use ``container-infra``. Other values are accepted such + as ``volume`` for cinder endpoints, ``object-store`` for swift, and so + on; + +* ``url_path``: mandatory attribute. It defines the path of the request + that we execute on the endpoint to gather data. For example, to gather + data from magnum, one can use ``v1/clusters/detail``; + +* ``metadata_fields``: optional field. It is a list of all fields that + the response of the request executed with ``url_path`` that we want to + retrieve. As an example, for magnum, one can use the following values: + + .. code-block:: yaml + + metadata_fields: + - "labels" + - "updated_at" + - "keypair" + - "master_flavor_id" + - "api_address" + - "master_addresses" + - "node_count" + - "docker_volume_size" + - "master_count" + - "node_addresses" + - "status_reason" + - "coe_version" + - "cluster_template_id" + - "name" + - "stack_id" + - "created_at" + - "discovery_url" + - "container_version" + +* ``skip_sample_values``: optional field. It defines the values that + might come in the ``value_attribute`` that we want to ignore. For + magnun, one could for instance, ignore some of the status it has for + clusters. Therefore, data is not gathered for clusters in the defined + status. + + .. code-block:: yaml + + skip_sample_values: + - "CREATE_FAILED" + - "DELETE_FAILED" + +* ``value_mapping``: optional attribute. It defines a mapping for the + values that the dynamic pollster is handling. This is the actual value + that is sent to Gnocchi or other backends. If there is no mapping + specified, we will use the raw value that is obtained with the use of + ``value_attribute``. An example for magnum, one can use: + + .. code-block:: yaml + + value_mapping: + CREATE_IN_PROGRESS: "0" + CREATE_FAILED: "1" + CREATE_COMPLETE: "2" + UPDATE_IN_PROGRESS: "3" + UPDATE_FAILED: "4" + UPDATE_COMPLETE: "5" + DELETE_IN_PROGRESS: "6" + DELETE_FAILED: "7" + DELETE_COMPLETE: "8" + RESUME_COMPLETE: "9" + RESUME_FAILED: "10" + RESTORE_COMPLETE: "11" + ROLLBACK_IN_PROGRESS: "12" + ROLLBACK_FAILED: "13" + ROLLBACK_COMPLETE: "14" + SNAPSHOT_COMPLETE: "15" + CHECK_COMPLETE: "16" + ADOPT_COMPLETE: "17" + +* ``default_value``: optional parameter. The default value for + the value mapping in case the variable value receives data that is not + mapped to something in the ``value_mapping`` configuration. This + attribute is only used when ``value_mapping`` is defined. Moreover, it + has a default of ``-1``. + +* ``metadata_mapping``: the map used to create new metadata fields. The key + is a metadata name that exists in the response of the request we make, + and the value of this map is the new desired metadata field that will be + created with the content of the metadata that we are mapping. + The ``metadata_mapping`` can be created as follows: + + .. code-block:: yaml + + metadata_mapping: + name: "display_name" + some_attribute: "new_attribute_name" + +* ``preserve_mapped_metadata``: indicates if we preserve the old metadata name + when it gets mapped to a new one. The default value is ``True``. + +The complete YAML configuration to gather data from Magnum (that has been used +as an example) is the following: + +.. code-block:: yaml + + --- + + - name: "dynamic.magnum.cluster" + sample_type: "gauge" + unit: "cluster" + value_attribute: "status" + endpoint_type: "container-infra" + url_path: "v1/clusters/detail" + metadata_fields: + - "labels" + - "updated_at" + - "keypair" + - "master_flavor_id" + - "api_address" + - "master_addresses" + - "node_count" + - "docker_volume_size" + - "master_count" + - "node_addresses" + - "status_reason" + - "coe_version" + - "cluster_template_id" + - "name" + - "stack_id" + - "created_at" + - "discovery_url" + - "container_version" + value_mapping: + CREATE_IN_PROGRESS: "0" + CREATE_FAILED: "1" + CREATE_COMPLETE: "2" + UPDATE_IN_PROGRESS: "3" + UPDATE_FAILED: "4" + UPDATE_COMPLETE: "5" + DELETE_IN_PROGRESS: "6" + DELETE_FAILED: "7" + DELETE_COMPLETE: "8" + RESUME_COMPLETE: "9" + RESUME_FAILED: "10" + RESTORE_COMPLETE: "11" + ROLLBACK_IN_PROGRESS: "12" + ROLLBACK_FAILED: "13" + ROLLBACK_COMPLETE: "14" + SNAPSHOT_COMPLETE: "15" + CHECK_COMPLETE: "16" + ADOPT_COMPLETE: "17" + +We can also replicate and enhance some hardcoded pollsters. +For instance, the pollster to gather VPN connections. Currently, +it is always persisting `1` for all of the VPN connections it finds. +However, the VPN connection can have multiple statuses, and we should +normally only bill for active resources, and not resources on `ERROR` +states. An example to gather VPN connections data is the following +(this is just an example, and one can adapt and configure as he/she +desires): + +.. code-block:: yaml + + --- + + - name: "dynamic.network.services.vpn.connection" + sample_type: "gauge" + unit: "ipsec_site_connection" + value_attribute: "status" + endpoint_type: "network" + url_path: "v2.0/vpn/ipsec-site-connections" + metadata_fields: + - "name" + - "vpnservice_id" + - "description" + - "status" + - "peer_address" + value_mapping: + ACTIVE: "1" + metadata_mapping: + name: "display_name" + default_value: 0 diff --git a/releasenotes/notes/dynamic-pollster-system-6b45c8c973201b2b.yaml b/releasenotes/notes/dynamic-pollster-system-6b45c8c973201b2b.yaml new file mode 100644 index 0000000000..cf7e7d8015 --- /dev/null +++ b/releasenotes/notes/dynamic-pollster-system-6b45c8c973201b2b.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add dynamic pollster system. The dynamic pollster system enables operators + to gather new metrics on the fly (without needing to code pollsters). \ No newline at end of file