diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py index 53c30e4294..bb45b85f90 100644 --- a/ceilometer/polling/dynamic_pollster.py +++ b/ceilometer/polling/dynamic_pollster.py @@ -19,6 +19,7 @@ """ import copy import re +import time from oslo_log import log @@ -108,20 +109,37 @@ class PollsterSampleExtractor(object): LOG.debug("Removed key [%s] with value [%s] from " "metadata set that is sent to Gnocchi.", k, k_value) - def generate_sample(self, pollster_sample, pollster_definitons=None): + def generate_sample( + self, pollster_sample, pollster_definitions=None, **kwargs): + pollster_definitions =\ - pollster_definitons or self.definitions.configurations + pollster_definitions or self.definitions.configurations metadata = dict() if 'metadata_fields' in pollster_definitions: for k in pollster_definitions['metadata_fields']: - val = self.retrieve_attribute_nested_value(pollster_sample, k) + val = self.retrieve_attribute_nested_value( + pollster_sample, value_attribute=k, + definitions=self.definitions.configurations) LOG.debug("Assigning value [%s] to metadata key [%s].", val, k) metadata[k] = val self.generate_new_metadata_fields( metadata=metadata, pollster_definitions=pollster_definitions) + + extra_metadata = self.definitions.retrieve_extra_metadata( + kwargs['manager'], pollster_sample) + + for key in extra_metadata.keys(): + if key in metadata.keys(): + LOG.warning("The extra metadata key [%s] already exist in " + "pollster current metadata set [%s]. Therefore, " + "we will ignore it with its value [%s].", + key, metadata, extra_metadata[key]) + continue + metadata[key] = extra_metadata[key] + return ceilometer_sample.Sample( timestamp=ceilometer_utils.isotime(), name=pollster_definitions['name'], @@ -134,15 +152,18 @@ class PollsterSampleExtractor(object): resource_metadata=metadata) def retrieve_attribute_nested_value(self, json_object, - value_attribute=None): + value_attribute=None, + definitions=None, **kwargs): + if not definitions: + definitions = self.definitions.configurations - attribute_key = value_attribute or self.definitions.\ - extract_attribute_key() + attribute_key = value_attribute + if not attribute_key: + attribute_key = self.definitions.extract_attribute_key() LOG.debug( "Retrieving the nested keys [%s] from [%s] or pollster [""%s].", - attribute_key, json_object, - self.definitions.configurations["name"]) + attribute_key, json_object, definitions["name"]) keys_and_operations = attribute_key.split("|") attribute_key = keys_and_operations[0].strip() @@ -153,9 +174,9 @@ class PollsterSampleExtractor(object): nested_keys = attribute_key.split(".") value = reduce(operator.getitem, nested_keys, json_object) - return self.operate_value(keys_and_operations, value) + return self.operate_value(keys_and_operations, value, definitions) - def operate_value(self, keys_and_operations, value): + def operate_value(self, keys_and_operations, value, definitions): # We do not have operations to be executed against the value extracted if len(keys_and_operations) < 2: return value @@ -164,24 +185,23 @@ class PollsterSampleExtractor(object): if 'value' not in operation: raise declarative.DynamicPollsterDefinitionException( "The attribute field operation [%s] must use the [" - "value] variable." % operation, - self.definitions.configurations) + "value] variable." % operation, definitions) LOG.debug("Executing operation [%s] against value[%s] for " "pollster [%s].", operation, value, - self.definitions.configurations["name"]) + definitions["name"]) value = eval(operation.strip()) - LOG.debug( - "Result [%s] of operation [%s] for pollster [%s].", - value, operation, self.definitions.configurations["name"]) + LOG.debug("Result [%s] of operation [%s] for pollster [%s].", + value, operation, definitions["name"]) return value class SimplePollsterSampleExtractor(PollsterSampleExtractor): - def generate_single_sample(self, pollster_sample): - value = self.retrieve_attribute_nested_value(pollster_sample) + def generate_single_sample(self, pollster_sample, **kwargs): + value = self.retrieve_attribute_nested_value( + pollster_sample) value = self.definitions.value_mapper.map_or_skip_value( value, pollster_sample) @@ -190,10 +210,10 @@ class SimplePollsterSampleExtractor(PollsterSampleExtractor): pollster_sample['value'] = value - return self.generate_sample(pollster_sample) + return self.generate_sample(pollster_sample, **kwargs) - def extract_sample(self, pollster_sample): - sample = self.generate_single_sample(pollster_sample) + def extract_sample(self, pollster_sample, **kwargs): + sample = self.generate_single_sample(pollster_sample, **kwargs) if isinstance(sample, SkippedSample): return sample yield sample @@ -201,9 +221,11 @@ class SimplePollsterSampleExtractor(PollsterSampleExtractor): class MultiMetricPollsterSampleExtractor(PollsterSampleExtractor): - def extract_sample(self, pollster_sample): + def extract_sample(self, pollster_sample, **kwargs): pollster_definitions = self.definitions.configurations - value = self.retrieve_attribute_nested_value(pollster_sample) + + value = self.retrieve_attribute_nested_value( + pollster_sample, definitions=pollster_definitions) LOG.debug("We are dealing with a multi metric pollster. The " "value we are processing is the following: [%s].", value) @@ -223,12 +245,12 @@ class MultiMetricPollsterSampleExtractor(PollsterSampleExtractor): pollster_name, value_attribute, sub_metric_placeholder, pollster_definitions, - pollster_sample) + pollster_sample, **kwargs) def extract_sub_samples(self, value, sub_metric_attribute_name, pollster_name, value_attribute, sub_metric_placeholder, pollster_definitions, - pollster_sample): + pollster_sample, **kwargs): for sub_sample in value: sub_metric_name = sub_sample[sub_metric_attribute_name] @@ -237,7 +259,7 @@ class MultiMetricPollsterSampleExtractor(PollsterSampleExtractor): pollster_definitions['name'] = new_metric_name actual_value = self.retrieve_attribute_nested_value( - sub_sample, value_attribute) + sub_sample, value_attribute, definitions=pollster_definitions) pollster_sample['value'] = actual_value @@ -245,7 +267,8 @@ class MultiMetricPollsterSampleExtractor(PollsterSampleExtractor): sub_metric_name): continue - yield self.generate_sample(pollster_sample, pollster_definitions) + yield self.generate_sample( + pollster_sample, pollster_definitions, **kwargs) def extract_field_name_from_value_attribute_configuration(self): value_attribute = self.definitions.configurations['value_attribute'] @@ -392,6 +415,8 @@ class PollsterDefinitions(object): POLLSTER_VALID_NAMES_REGEXP = r"^([\w-]+)(\.[\w-]+)*(\.{[\w-]+})?$" + EXTERNAL_ENDPOINT_TYPE = "external" + standard_definitions = [ PollsterDefinition(name='name', required=True, validation_regex=POLLSTER_VALID_NAMES_REGEXP), @@ -412,7 +437,11 @@ class PollsterDefinitions(object): PollsterDefinition(name='resource_id_attribute', default="id"), PollsterDefinition(name='project_id_attribute', default="project_id"), PollsterDefinition(name='headers'), - PollsterDefinition(name='timeout', default=30)] + PollsterDefinition(name='timeout', default=30), + PollsterDefinition(name='extra_metadata_fields_cache_seconds', + default=3600), + PollsterDefinition(name='extra_metadata_fields') + ] extra_definitions = [] @@ -424,6 +453,7 @@ class PollsterDefinitions(object): self.validate_missing() self.sample_gatherer = PollsterSampleGatherer(self) self.sample_extractor = SimplePollsterSampleExtractor(self) + self.response_cache = {} def validate_configurations(self, configurations): for k, v in self.definitions.items(): @@ -464,6 +494,115 @@ class PollsterDefinitions(object): "Required fields %s not specified." % missing, self.configurations) + def retrieve_extra_metadata(self, manager, request_sample): + extra_metadata_fields = self.configurations['extra_metadata_fields'] + if extra_metadata_fields: + if isinstance(self, NonOpenStackApisPollsterDefinition): + raise declarative.NonOpenStackApisDynamicPollsterException( + "Not supported the use of extra metadata gathering for " + "non-openstack pollsters [%s] (yet)." + % self.configurations['name']) + + return self._retrieve_extra_metadata( + extra_metadata_fields, manager, request_sample) + + LOG.debug("No extra metadata to be captured for pollsters [%s] and " + "request sample [%s].", self.definitions, request_sample) + return {} + + def _retrieve_extra_metadata( + self, extra_metadata_fields, manager, request_sample): + LOG.debug("Processing extra metadata fields [%s] for " + "sample [%s].", extra_metadata_fields, + request_sample) + + extra_metadata_captured = {} + for extra_metadata in extra_metadata_fields: + extra_metadata_name = extra_metadata['name'] + + if extra_metadata_name in extra_metadata_captured.keys(): + LOG.warning("Duplicated extra metadata name [%s]. Therefore, " + "we do not process this iteration [%s].", + extra_metadata_name, extra_metadata) + continue + + LOG.debug("Processing extra metadata [%s] for sample [%s].", + extra_metadata_name, request_sample) + + endpoint_type = 'endpoint:' + extra_metadata['endpoint_type'] + if not endpoint_type.endswith( + PollsterDefinitions.EXTERNAL_ENDPOINT_TYPE): + response = self.execute_openstack_extra_metadata_gathering( + endpoint_type, extra_metadata, manager, request_sample, + extra_metadata_captured) + else: + raise declarative.NonOpenStackApisDynamicPollsterException( + "Not supported the use of extra metadata gathering for " + "non-openstack endpoints [%s] (yet)." % extra_metadata) + + extra_metadata_extractor_kwargs = { + 'value_attribute': extra_metadata['value'], + 'sample': request_sample} + + extra_metadata_value = \ + self.sample_extractor.retrieve_attribute_nested_value( + response, **extra_metadata_extractor_kwargs) + + LOG.debug("Generated extra metadata [%s] with value [%s].", + extra_metadata_name, extra_metadata_value) + extra_metadata_captured[extra_metadata_name] = extra_metadata_value + + return extra_metadata_captured + + def execute_openstack_extra_metadata_gathering(self, endpoint_type, + extra_metadata, manager, + request_sample, + extra_metadata_captured): + url_for_endpoint_type = manager.discover( + [endpoint_type], self.response_cache) + + LOG.debug("URL [%s] found for endpoint type [%s].", + url_for_endpoint_type, endpoint_type) + + if url_for_endpoint_type: + url_for_endpoint_type = url_for_endpoint_type[0] + + self.sample_gatherer.generate_url_path( + extra_metadata, request_sample, extra_metadata_captured) + + cached_response, max_ttl_for_cache = self.response_cache.get( + extra_metadata['url_path'], (None, None)) + + extra_metadata_fields_cache_seconds = extra_metadata.get( + 'extra_metadata_fields_cache_seconds', + self.configurations['extra_metadata_fields_cache_seconds']) + + current_time = time.time() + if cached_response and max_ttl_for_cache >= current_time: + LOG.debug("Returning response [%s] for request [%s] as the TTL " + "[max=%s, current_time=%s] has not expired yet.", + cached_response, extra_metadata['url_path'], + max_ttl_for_cache, current_time) + return cached_response + + if cached_response: + LOG.debug("Cleaning cached response [%s] for request [%s] " + "as the TTL [max=%s, current_time=%s] has expired.", + cached_response, extra_metadata['url_path'], + max_ttl_for_cache, current_time) + + response = self.sample_gatherer.execute_request_for_definitions( + extra_metadata, **{'manager': manager, + 'keystone_client': manager._keystone, + 'resource': url_for_endpoint_type, + 'execute_id_overrides': False}) + + max_ttl_for_cache = time.time() + extra_metadata_fields_cache_seconds + + cache_tuple = (response, max_ttl_for_cache) + self.response_cache[extra_metadata['url_path']] = cache_tuple + return response + class MultiMetricPollsterDefinitions(PollsterDefinitions): @@ -522,36 +661,45 @@ class PollsterSampleGatherer(object): return 'endpoint:' + self.definitions.configurations['endpoint_type'] def execute_request_get_samples(self, **kwargs): - resp, url = self.definitions.sample_gatherer. \ - internal_execute_request_get_samples(kwargs) + return self.execute_request_for_definitions( + self.definitions.configurations, **kwargs) + + def execute_request_for_definitions(self, definitions, **kwargs): + resp, url = self._internal_execute_request_get_samples( + definitions=definitions, **kwargs) response_json = resp.json() entry_size = len(response_json) LOG.debug("Entries [%s] in the JSON for request [%s] " "for dynamic pollster [%s].", - response_json, url, self.definitions.configurations['name']) + response_json, url, definitions['name']) if entry_size > 0: - samples = self.retrieve_entries_from_response(response_json) - url_to_next_sample = self.get_url_to_next_sample(response_json) + samples = self.retrieve_entries_from_response( + response_json, definitions) + url_to_next_sample = self.get_url_to_next_sample( + response_json, definitions) + + self.prepare_samples(definitions, samples, **kwargs) + if url_to_next_sample: kwargs['next_sample_url'] = url_to_next_sample - samples += self.execute_request_get_samples(**kwargs) - - self.execute_id_overrides(samples) + samples += self.execute_request_for_definitions( + definitions=definitions, **kwargs) return samples return [] - def execute_id_overrides(self, samples): - if samples: - user_id_attribute = self.definitions.configurations[ - 'user_id_attribute'] - project_id_attribute = self.definitions.configurations[ - 'project_id_attribute'] - resource_id_attribute = self.definitions.configurations[ - 'resource_id_attribute'] - + def prepare_samples( + self, definitions, samples, execute_id_overrides=True, **kwargs): + if samples and execute_id_overrides: for request_sample in samples: + user_id_attribute = definitions.get( + 'user_id_attribute', 'user_id') + project_id_attribute = definitions.get( + 'project_id_attribute', 'project_id') + resource_id_attribute = definitions.get( + 'resource_id_attribute', 'id') + self.generate_new_attributes_in_sample( request_sample, user_id_attribute, 'user_id') self.generate_new_attributes_in_sample( @@ -559,6 +707,21 @@ class PollsterSampleGatherer(object): self.generate_new_attributes_in_sample( request_sample, resource_id_attribute, 'id') + def generate_url_path(self, extra_metadata, sample, + extra_metadata_captured): + if not extra_metadata.get('url_path_original'): + extra_metadata[ + 'url_path_original'] = extra_metadata['url_path'] + + extra_metadata['url_path'] = eval( + extra_metadata['url_path_original']) + + LOG.debug("URL [%s] generated for pattern [%s] for sample [%s] and " + "extra metadata captured [%s].", + extra_metadata['url_path'], + extra_metadata['url_path_original'], sample, + extra_metadata_captured) + def generate_new_attributes_in_sample( self, sample, attribute_key, new_attribute_key): @@ -578,9 +741,9 @@ class PollsterSampleGatherer(object): sample[new_attribute_key] = attribute_value - def get_url_to_next_sample(self, resp): - linked_sample_extractor = self.definitions.configurations[ - 'next_sample_url_attribute'] + def get_url_to_next_sample(self, resp, definitions): + linked_sample_extractor = definitions.get('next_sample_url_attribute') + if not linked_sample_extractor: return None @@ -592,37 +755,40 @@ class PollsterSampleGatherer(object): "the configuration [%s]", resp, linked_sample_extractor) return None - def internal_execute_request_get_samples(self, kwargs): - keystone_client = kwargs['keystone_client'] - url = self.get_request_linked_samples_url(kwargs) + def _internal_execute_request_get_samples(self, definitions=None, + keystone_client=None, **kwargs): + if not definitions: + definitions = self.definitions.configurations - request_arguments = self.create_request_arguments() + url = self.get_request_linked_samples_url(kwargs, definitions) + request_arguments = self.create_request_arguments(definitions) LOG.debug("Executing request against [url=%s] with parameters [" "%s] for pollsters [%s]", url, request_arguments, - self.definitions.configurations["name"]) - + definitions["name"]) resp = keystone_client.session.get(url, **request_arguments) - if resp.status_code != requests.codes.ok: resp.raise_for_status() return resp, url - def create_request_arguments(self): + def create_request_arguments(self, definitions): request_args = { "authenticated": True } - request_headers = self.definitions.configurations['headers'] + request_headers = definitions.get('headers', []) if request_headers: request_args['headers'] = request_headers - request_args['timeout'] = self.definitions.configurations['timeout'] + request_args['timeout'] = definitions.get('timeout', 300) return request_args - def get_request_linked_samples_url(self, kwargs): + def get_request_linked_samples_url(self, kwargs, definitions): next_sample_url = kwargs.get('next_sample_url') if next_sample_url: return self.get_next_page_url(kwargs, next_sample_url) + + LOG.debug("Generating url with [%s] and path [%s].", + kwargs, definitions['url_path']) return self.get_request_url( - kwargs, self.definitions.configurations['url_path']) + kwargs, definitions['url_path']) def get_next_page_url(self, kwargs, next_sample_url): parse_result = urlparse.urlparse(next_sample_url) @@ -634,19 +800,19 @@ class PollsterSampleGatherer(object): endpoint = kwargs['resource'] return urlparse.urljoin(endpoint, url_path) - def retrieve_entries_from_response(self, response_json): + def retrieve_entries_from_response(self, response_json, definitions): if isinstance(response_json, list): return response_json - first_entry_name = \ - self.definitions.configurations['response_entries_key'] + first_entry_name = definitions.get('response_entries_key') + if not first_entry_name: try: first_entry_name = next(iter(response_json)) except RuntimeError as e: LOG.debug("Generator threw a StopIteration " "and we need to catch it [%s].", e) - return self.definitions.sample_extractor. \ + return self.definitions.sample_extractor.\ retrieve_attribute_nested_value(response_json, first_entry_name) @@ -677,19 +843,17 @@ class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer): return 'barbican:' + \ self.definitions.configurations['barbican_secret_id'] - def internal_execute_request_get_samples(self, kwargs): + def _internal_execute_request_get_samples(self, definitions, **kwargs): credentials = kwargs['resource'] - override_credentials = self.definitions.configurations[ - 'authentication_parameters'] + override_credentials = definitions['authentication_parameters'] if override_credentials: credentials = override_credentials - url = self.get_request_linked_samples_url(kwargs) + url = self.get_request_linked_samples_url(kwargs, definitions) - authenticator_module_name = self.definitions.configurations['module'] - authenticator_class_name = \ - self.definitions.configurations['authentication_object'] + authenticator_module_name = definitions['module'] + authenticator_class_name = definitions['authentication_object'] imported_module = __import__(authenticator_module_name) authenticator_class = getattr(imported_module, @@ -698,12 +862,12 @@ class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer): authenticator_arguments = list(map(str.strip, credentials.split(","))) authenticator_instance = authenticator_class(*authenticator_arguments) - request_arguments = self.create_request_arguments() + request_arguments = self.create_request_arguments(definitions) request_arguments["auth"] = authenticator_instance LOG.debug("Executing request against [url=%s] with parameters [" "%s] for pollsters [%s]", url, request_arguments, - self.definitions.configurations["name"]) + definitions["name"]) resp = requests.get(url, **request_arguments) if resp.status_code != requests.codes.ok: @@ -714,9 +878,10 @@ class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer): return resp, url - def create_request_arguments(self): + def create_request_arguments(self, definitions): request_arguments = super( - NonOpenStackApisSamplesGatherer, self).create_request_arguments() + NonOpenStackApisSamplesGatherer, self).create_request_arguments( + definitions) request_arguments.pop("authenticated") @@ -728,28 +893,6 @@ class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer): return url_path return urlparse.urljoin(endpoint, url_path) - def execute_request_get_samples(self, **kwargs): - samples = super(NonOpenStackApisSamplesGatherer, - self).execute_request_get_samples(**kwargs) - - if samples: - user_id_attribute = self.definitions.configurations[ - 'user_id_attribute'] - project_id_attribute = self.definitions.configurations[ - 'project_id_attribute'] - resource_id_attribute = self.definitions.configurations[ - 'resource_id_attribute'] - - for request_sample in samples: - self.generate_new_attributes_in_sample( - request_sample, user_id_attribute, 'user_id') - self.generate_new_attributes_in_sample( - request_sample, project_id_attribute, 'project_id') - self.generate_new_attributes_in_sample( - request_sample, resource_id_attribute, 'id') - - return samples - def generate_new_attributes_in_sample( self, sample, attribute_key, new_attribute_key): if attribute_key: @@ -796,8 +939,9 @@ class DynamicPollster(plugin_base.PollsterBase): def load_samples(self, resource, manager): try: return self.definitions.sample_gatherer.\ - execute_request_get_samples(keystone_client=manager._keystone, - resource=resource) + execute_request_get_samples(manager=manager, + resource=resource, + keystone_client=manager._keystone) except RequestException as e: LOG.warning("Error [%s] while loading samples for [%s] " "for dynamic pollster [%s].", @@ -814,11 +958,12 @@ class DynamicPollster(plugin_base.PollsterBase): LOG.debug("Executing get sample for resource [%s].", r) samples = self.load_samples(r, manager) for pollster_sample in samples: - sample = self.extract_sample(pollster_sample) + kwargs = {'manager': manager, 'resource': r} + sample = self.extract_sample(pollster_sample, **kwargs) if isinstance(sample, SkippedSample): continue yield from sample - def extract_sample(self, pollster_sample): + def extract_sample(self, pollster_sample, **kwargs): return self.definitions.sample_extractor.extract_sample( - pollster_sample) + pollster_sample, **kwargs) diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py index fa5d8baea1..e596f7b517 100644 --- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py @@ -463,7 +463,7 @@ class TestDynamicPollster(base.BaseTestCase): self.assertEqual(0, len(samples_list)) - def fake_sample_list(self, keystone_client=None, resource=None): + def fake_sample_list(self, **kwargs): samples_list = list() samples_list.append( {'name': "sample5", 'volume': 5, 'description': "desc-sample-5", @@ -520,7 +520,7 @@ class TestDynamicPollster(base.BaseTestCase): response = [{"object1-attr1": 1}, {"object1-attr2": 2}] entries = pollster.definitions.sample_gatherer. \ - retrieve_entries_from_response(response) + retrieve_entries_from_response(response, pollster.definitions) self.assertEqual(response, entries) @@ -538,8 +538,9 @@ class TestDynamicPollster(base.BaseTestCase): response = {"first": first_entries_from_response, "second": second_entries_from_response} - entries = pollster.definitions.sample_gatherer. \ - retrieve_entries_from_response(response) + entries = pollster.definitions.sample_gatherer.\ + retrieve_entries_from_response( + response, pollster.definitions.configurations) self.assertEqual(first_entries_from_response, entries) @@ -557,7 +558,8 @@ class TestDynamicPollster(base.BaseTestCase): response = {"first": first_entries_from_response, "second": second_entries_from_response} entries = pollster.definitions.sample_gatherer. \ - retrieve_entries_from_response(response) + retrieve_entries_from_response(response, + pollster.definitions.configurations) self.assertEqual(second_entries_from_response, entries) @@ -640,7 +642,7 @@ class TestDynamicPollster(base.BaseTestCase): self.assertEqual(expected_value_after_operations, returned_value) - def fake_sample_multi_metric(self, keystone_client=None, resource=None): + def fake_sample_multi_metric(self, **kwargs): multi_metric_sample_list = [ {"categories": [ { @@ -724,17 +726,17 @@ class TestDynamicPollster(base.BaseTestCase): 'project_id': "2334", 'id': "35"} - def internal_execute_request_get_samples_mock(self, arg): + def internal_execute_request_get_samples_mock(self, **kwargs): class Response: def json(self): return [sample] return Response(), "url" original_method = dynamic_pollster.PollsterSampleGatherer.\ - internal_execute_request_get_samples + _internal_execute_request_get_samples try: dynamic_pollster.PollsterSampleGatherer. \ - internal_execute_request_get_samples = \ + _internal_execute_request_get_samples = \ internal_execute_request_get_samples_mock self.pollster_definition_all_fields[ @@ -759,7 +761,7 @@ class TestDynamicPollster(base.BaseTestCase): response[0]['id']) finally: dynamic_pollster.PollsterSampleGatherer. \ - internal_execute_request_get_samples = original_method + _internal_execute_request_get_samples = original_method def test_retrieve_attribute_self_reference_sample(self): key = " . | value['key1']['subKey1'][0]['d'] if 'key1' in value else 0" @@ -795,7 +797,7 @@ class TestDynamicPollster(base.BaseTestCase): pollster = dynamic_pollster.DynamicPollster(pollster_definition) request_args = pollster.definitions.sample_gatherer\ - .create_request_arguments() + .create_request_arguments(pollster.definitions.configurations) self.assertTrue("headers" in request_args) self.assertEqual(2, len(request_args["headers"])) @@ -821,7 +823,7 @@ class TestDynamicPollster(base.BaseTestCase): pollster = dynamic_pollster.DynamicPollster(pollster_definition) request_args = pollster.definitions.sample_gatherer\ - .create_request_arguments() + .create_request_arguments(pollster.definitions.configurations) self.assertTrue("headers" in request_args) self.assertTrue("authenticated" in request_args) @@ -843,7 +845,8 @@ class TestDynamicPollster(base.BaseTestCase): self.pollster_definition_only_required_fields) request_args =\ - pollster.definitions.sample_gatherer.create_request_arguments() + pollster.definitions.sample_gatherer.create_request_arguments( + pollster.definitions.configurations) self.assertTrue("headers" not in request_args) self.assertTrue("authenticated" in request_args) @@ -902,7 +905,8 @@ class TestDynamicPollster(base.BaseTestCase): kwargs = {'resource': base_url} url = pollster.definitions.sample_gatherer\ - .get_request_linked_samples_url(kwargs) + .get_request_linked_samples_url( + kwargs, pollster.definitions.configurations) self.assertEqual(expected_url, url) @@ -917,7 +921,7 @@ class TestDynamicPollster(base.BaseTestCase): 'next_sample_url': expected_url} url = pollster.definitions.sample_gatherer\ - .get_request_linked_samples_url(kwargs) + .get_request_linked_samples_url(kwargs, pollster.definitions) self.assertEqual(expected_url, url) @@ -932,7 +936,8 @@ class TestDynamicPollster(base.BaseTestCase): 'next_sample_url': "/next_page"} url = pollster.definitions.sample_gatherer\ - .get_request_linked_samples_url(kwargs) + .get_request_linked_samples_url( + kwargs, pollster.definitions.configurations) self.assertEqual(expected_url, url) @@ -947,7 +952,8 @@ class TestDynamicPollster(base.BaseTestCase): 'value': 1} sample = pollster.definitions.sample_extractor.generate_sample( - pollster_sample) + pollster_sample, pollster.definitions.configurations, + manager=mock.Mock()) self.assertEqual(1, sample.volume) self.assertEqual(2, len(sample.resource_metadata)) @@ -968,7 +974,8 @@ class TestDynamicPollster(base.BaseTestCase): 'value': 1} sample = pollster.definitions.sample_extractor.generate_sample( - pollster_sample) + pollster_sample, pollster.definitions.configurations, + manager=mock.Mock()) self.assertEqual(1, sample.volume) self.assertEqual(3, len(sample.resource_metadata)) @@ -990,7 +997,8 @@ class TestDynamicPollster(base.BaseTestCase): 'value': 1} sample = pollster.definitions.sample_extractor.generate_sample( - pollster_sample) + pollster_sample, pollster.definitions.configurations, + manager=mock.Mock()) self.assertEqual(1, sample.volume) self.assertEqual(3, len(sample.resource_metadata)) diff --git a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py index d9ba22050a..d8f32ff3fe 100644 --- a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py @@ -47,7 +47,7 @@ OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values', ALL_POLLSTER_FIELDS = REQUIRED_POLLSTER_FIELDS + OPTIONAL_POLLSTER_FIELDS -def fake_sample_multi_metric(self, keystone_client=None, resource=None): +def fake_sample_multi_metric(self, **kwargs): multi_metric_sample_list = [ {"user_id": "UID-U007", "project_id": "UID-P007", @@ -236,8 +236,9 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): kwargs = {'resource': "credentials"} - resp, url = pollster.definitions.sample_gatherer. \ - internal_execute_request_get_samples(kwargs) + resp, url = pollster.definitions.sample_gatherer.\ + _internal_execute_request_get_samples( + pollster.definitions.configurations, **kwargs) self.assertEqual( self.pollster_definition_only_required_fields['url_path'], url) @@ -265,7 +266,8 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): exception = self.assertRaises( NonOpenStackApisDynamicPollsterException, pollster.definitions.sample_gatherer. - internal_execute_request_get_samples, kwargs) + _internal_execute_request_get_samples, + pollster.definitions.configurations, **kwargs) self.assertEqual( "NonOpenStackApisDynamicPollsterException" @@ -307,17 +309,18 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): 'project_id_attribute': "dfghyt432345t", 'resource_id_attribute': "sdfghjt543"} - def internal_execute_request_get_samples_mock(self, arg): + def internal_execute_request_get_samples_mock( + self, definitions, **kwargs): class Response: def json(self): return [sample] return Response(), "url" original_method = NonOpenStackApisSamplesGatherer. \ - internal_execute_request_get_samples + _internal_execute_request_get_samples try: NonOpenStackApisSamplesGatherer. \ - internal_execute_request_get_samples = \ + _internal_execute_request_get_samples = \ internal_execute_request_get_samples_mock self.pollster_definition_all_fields[ @@ -342,7 +345,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): response[0]['id']) finally: NonOpenStackApisSamplesGatherer. \ - internal_execute_request_get_samples = original_method + _internal_execute_request_get_samples = original_method def test_execute_request_get_samples_empty_keys(self): sample = {'user_id_attribute': "123456789", @@ -446,7 +449,8 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): kwargs = {'resource': "non-openstack-resource"} url = pollster.definitions.sample_gatherer\ - .get_request_linked_samples_url(kwargs) + .get_request_linked_samples_url( + kwargs, pollster.definitions.configurations) self.assertEqual(expected_url, url) @@ -461,7 +465,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): kwargs = {'next_sample_url': expected_url} url = pollster.definitions.sample_gatherer\ - .get_request_linked_samples_url(kwargs) + .get_request_linked_samples_url(kwargs, pollster.definitions) self.assertEqual(expected_url, url) @@ -476,6 +480,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): kwargs = {'next_sample_url': next_sample_path} url = pollster.definitions.sample_gatherer\ - .get_request_linked_samples_url(kwargs) + .get_request_linked_samples_url( + kwargs, pollster.definitions.configurations) self.assertEqual(expected_url, url) diff --git a/doc/source/admin/telemetry-dynamic-pollster.rst b/doc/source/admin/telemetry-dynamic-pollster.rst index 9a321cb63e..d861c07f98 100644 --- a/doc/source/admin/telemetry-dynamic-pollster.rst +++ b/doc/source/admin/telemetry-dynamic-pollster.rst @@ -746,3 +746,148 @@ presented as follows: url_path: "v1/test-volumes" response_entries_key: "servers" next_sample_url_attribute: "server_link | filter(lambda v: v.get('rel') == 'next', value) | list(value) | value[0] | value.get('href')" + +OpenStack Dynamic pollsters metadata enrichment with other OpenStack API's data +------------------------------------------------------------------------------- + +Sometimes we want/need to add/gather extra metadata for the samples being +handled by Ceilometer Dynamic pollsters, such as the project name, domain id, +domain name, and other metadata that are not always accessible via the +OpenStack component where the sample is gathered. + +For instance, when gathering the status of virtual machines (VMs) from Nova, +we only have the `tenant_id`, which must be used as the `project_id`. However, +for billing and later invoicing one might need/want the project name, domain +id, and other metadata that are available in Keystone (and maybe some others +that are scattered over other components). To achieve that, one can use the +OpenStack metadata enrichment option. This feature is only available +to *OpenStack pollsters*, and can only gather extra metadata from OpenStack +APIs. As follows we present an example that shows a dynamic pollster +configuration to gather virtual machine (VM) status, and to enrich the data +pushed to the storage backend (e.g. Gnocchi) with project name, domain ID, +and domain name. + + .. code-block:: yaml + + --- + + - name: "dynamic_pollster.instance.status" + next_sample_url_attribute: "server_links | filter(lambda v: v.get('rel') == 'next', value) | list(value) | value[0] | value.get('href') | value.replace('http:', 'https:')" + sample_type: "gauge" + unit: "server" + value_attribute: "status" + endpoint_type: "compute" + url_path: "/v2.1/servers/detail?all_tenants=true" + headers: + "Openstack-API-Version": "compute 2.65" + project_id_attribute: "tenant_id" + metadata_fields: + - "status" + - "name" + - "flavor.vcpus" + - "flavor.ram" + - "flavor.disk" + - "flavor.ephemeral" + - "flavor.swap" + - "flavor.original_name" + - "image | value or { 'id': '' } | value['id']" + - "OS-EXT-AZ:availability_zone" + - "OS-EXT-SRV-ATTR:host" + - "user_id" + - "tags | ','.join(value)" + - "locked" + value_mapping: + ACTIVE: "1" + default_value: 0 + metadata_mapping: + "OS-EXT-AZ:availability_zone": "dynamic_availability_zone" + "OS-EXT-SRV-ATTR:host": "dynamic_host" + "flavor.original_name": "dynamic_flavor_name" + "flavor.vcpus": "dynamic_flavor_vcpus" + "flavor.ram": "dynamic_flavor_ram" + "flavor.disk": "dynamic_flavor_disk" + "flavor.ephemeral": "dynamic_flavor_ephemeral" + "flavor.swap": "dynamic_flavor_swap" + "image | value or { 'id': '' } | value['id']": "dynamic_image_ref" + "name": "dynamic_display_name" + "locked": "dynamic_locked" + "tags | ','.join(value)": "dynamic_tags" + extra_metadata_fields_cache_seconds: 3600 + extra_metadata_fields: + - name: "project_name" + endpoint_type: "identity" + url_path: "'/v3/projects/' + str(sample['project_id'])" + headers: + "Openstack-API-Version": "identity latest" + value: "name" + extra_metadata_fields_cache_seconds: 1800 # overriding the default cache policy + - name: "domain_id" + endpoint_type: "identity" + url_path: "'/v3/projects/' + str(sample['project_id'])" + headers: + "Openstack-API-Version": "identity latest" + value: "domain_id" + - name: "domain_name" + endpoint_type: "identity" + url_path: "'/v3/domains/' + str(extra_metadata_captured['domain_id'])" + headers: + "Openstack-API-Version": "identity latest" + value: "name" + +The above example can be used to gather and persist in the backend the +status of VMs. It will persist `1` in the backend as a measure for every +collecting period if the VM's status is `ACTIVE`, and `0` otherwise. This is +quite useful to create hashmap rating rules for running VMs in CloudKitty. +Then, to enrich the resource in the storage backend, we are adding extra +metadata that are collected in Keystone via the `extra_metadata_fields` +options. + +The metadata enrichment feature has the following options: + +* ``extra_metadata_fields_cache_seconds``: optional parameter. Defines the + extra metadata request's response cache. Some requests, such as the ones + executed against Keystone to retrieve extra metadata are rather static. + Therefore, one does not need to constantly re-execute the request. That + is the reason why we cache the response of such requests. By default the + cache time to live (TTL) for responses is `3600` seconds. However, this + value can be increased of decreased. + +* ``extra_metadata_fields``: optional parameter. This option is a list of + objects, where each one of its elements is an extra metadata definition. + Each one of the extra metadata definition can have the options defined in + the dynamic pollsters such as to handle paged responses, operations on the + extracted values, headers and so on. The basic options that must be + defined for an extra metadata definitions are the following: + + * ``name``: This option is mandatory. The name of the extra metadata. + This is the name that is going to be used by the metadata. If there is + already any other metadata gathered via `metadata_fields` option or + transformed via `metadata_mapping` configuration, this metadata is + going to be discarded. + + * ``endpoint_type``: The endpoint type that we want to execute the + call against. This option is mandatory. It works similarly to the + `endpoint_type` option in the dynamic pollster definition. + + * ``url_path``: This option is mandatory. It works similarly to the + `url_path` option in the dynamic pollster definition. However, this + `one enables operators to execute/evaluate expressions in runtime, which + `allows one to retrieve the information from previously gathered + metadata via ``extra_metadata_captured` dictionary, or via the + `sample` itself. + + * ``value``: This configuration is mandatory. It works similarly to the + `value_attribute` option in the dynamic pollster definition. It is + the value we want to extract from the response, and assign in the + metadata being generated. + + * ``headers``: This option is optional. It works similarly to the + `headers` option in the dynamic pollster definition. + + * ``next_sample_url_attribute``: This option is optional. It works + similarly to the `next_sample_url_attribute` option in the dynamic + pollster definition. + + * ``response_entries_key``: This option is optional. It works + similarly to the `response_entries_key` option in the dynamic + pollster definition. diff --git a/releasenotes/notes/openstack-dynamic-pollsters-metadata-enrichment-703cf5914cf0c578.yaml b/releasenotes/notes/openstack-dynamic-pollsters-metadata-enrichment-703cf5914cf0c578.yaml new file mode 100644 index 0000000000..27c040b761 --- /dev/null +++ b/releasenotes/notes/openstack-dynamic-pollsters-metadata-enrichment-703cf5914cf0c578.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + OpenStack Dynamic pollsters metadata enrichment with other OpenStack API's data.