From 6f2acc6e3335ece57a190143c62f184c110ecce4 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 17 Jan 2020 16:43:43 -0300 Subject: [PATCH] Add support to linked samples responses Some APIs split their responses into pages and each response has a link to the next one. To allow operators to deal with this kind of APIs, we propose to extend the Dynamic pollsters to navigate through the API responses and join all responses into a single one. To enable it, the operator will need to configure the parameter `next_sample_url_attribute` with a mapper to the response's next page attribute. Change-Id: Ida0a73d2964f192e6c63a6b7e8003ef2b52bd710 --- ceilometer/polling/dynamic_pollster.py | 39 ++++- .../unit/polling/test_dynamic_pollster.py | 18 ++- ...est_non_openstack_credentials_discovery.py | 6 +- .../admin/telemetry-dynamic-pollster.rst | 140 +++++++++++++++++- 4 files changed, 187 insertions(+), 16 deletions(-) diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py index 913e30cd4d..0caab0fbef 100644 --- a/ceilometer/polling/dynamic_pollster.py +++ b/ceilometer/polling/dynamic_pollster.py @@ -361,7 +361,8 @@ class PollsterDefinitions(object): PollsterDefinition(name='default_value', default=-1), PollsterDefinition(name='metadata_mapping', default={}), PollsterDefinition(name='preserve_mapped_metadata', default=True), - PollsterDefinition(name='response_entries_key')] + PollsterDefinition(name='response_entries_key'), + PollsterDefinition(name='next_sample_url_attribute')] extra_definitions = [] @@ -481,19 +482,47 @@ class PollsterSampleGatherer(object): response_json, url, self.definitions.configurations['name']) if entry_size > 0: - return self.retrieve_entries_from_response(response_json) + response = self.retrieve_entries_from_response(response_json) + url_to_next_sample = self.get_url_to_next_sample(response_json) + if url_to_next_sample: + kwargs['next_sample_url'] = url_to_next_sample + response += self.execute_request_get_samples(**kwargs) + return response return [] + def get_url_to_next_sample(self, resp): + linked_sample_extractor = self.definitions.configurations[ + 'next_sample_url_attribute'] + if not linked_sample_extractor: + return None + + try: + return self.definitions.sample_extractor.\ + retrieve_attribute_nested_value(resp, linked_sample_extractor) + except KeyError: + LOG.debug("There is no next sample url for the sample [%s] using " + "the configuration [%s]", resp, linked_sample_extractor) + return None + def internal_execute_request_get_samples(self, kwargs): keystone_client = kwargs['keystone_client'] - endpoint = kwargs['resource'] - url = url_parse.urljoin( - endpoint, self.definitions.configurations['url_path']) + url = self.get_request_linked_samples_url(kwargs) resp = keystone_client.session.get(url, authenticated=True) if resp.status_code != requests.codes.ok: resp.raise_for_status() return resp, url + def get_request_linked_samples_url(self, kwargs): + next_sample_url = kwargs.get('next_sample_url') + if next_sample_url: + return next_sample_url + return self.get_request_url(kwargs) + + def get_request_url(self, kwargs): + endpoint = kwargs['resource'] + return url_parse.urljoin( + endpoint, self.definitions.configurations['url_path']) + def retrieve_entries_from_response(self, response_json): if isinstance(response_json, list): return response_json diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py index 912f7b8f8a..5118a82dc2 100644 --- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py @@ -78,7 +78,7 @@ class PagedSamplesGenerator(SampleGenerator): for page_link, page_size in page_links.items(): page_link = page_base_link + "/" + page_link self.response[current_page_link] = { - self.page_link_name: page_link, + self.page_link_name: [{'href': page_link, 'rel': 'next'}], self.dict_name: self.populate_page(page_size) } current_page_link = page_link @@ -158,7 +158,7 @@ class TestDynamicPollster(base.BaseTestCase): self.assertEqual(pollster_definition, pollster.pollster_definitions) @mock.patch('keystoneclient.v2_0.client.Client') - def test_skip_samples(self, keystone_mock): + def test_skip_samples_with_linked_samples(self, keystone_mock): generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={ 'volume': SampleGenerator(samples_dict={ 'name': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'], @@ -182,16 +182,26 @@ class TestDynamicPollster(base.BaseTestCase): pollster_definition['skip_sample_values'] = ['rb'] pollster_definition['url_path'] = 'v1/test-volumes' pollster_definition['response_entries_key'] = 'servers' + pollster_definition['next_sample_url_attribute'] = \ + 'server_link | filter(lambda v: v.get("rel") == "next", value) |' \ + 'list(value) | value[0] | value.get("href")' pollster = dynamic_pollster.DynamicPollster(pollster_definition) samples = pollster.get_samples(fake_manager, None, ['http://test.com']) - self.assertEqual(['ra', 'rc'], list(map(lambda s: s.volume, samples))) + self.assertEqual(['ra', 'rc', 'rd', 're', 'rf', 'rg', 'rh'], + list(map(lambda s: s.volume, samples))) + + generator.generate_samples('http://test.com/v1/test-volumes', { + 'marker=c3': 3, + 'marker=f6': 3 + }, 2) pollster_definition['name'] = 'test-pollster' pollster_definition['value_attribute'] = 'name' pollster_definition['skip_sample_values'] = ['b2'] pollster = dynamic_pollster.DynamicPollster(pollster_definition) samples = pollster.get_samples(fake_manager, None, ['http://test.com']) - self.assertEqual(['a1', 'c3'], list(map(lambda s: s.volume, samples))) + self.assertEqual(['a1', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8'], + list(map(lambda s: s.volume, samples))) def test_all_required_fields_ok(self): pollster = dynamic_pollster.DynamicPollster( diff --git a/ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py b/ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py index 33cb0dbf53..9559b9bc88 100644 --- a/ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py +++ b/ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py @@ -52,7 +52,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase): self.assertEqual(['No secrets found'], result) def test_discover_no_barbican_endpoint(self): - def discover_mock(self, type): + def discover_mock(self, manager, param=None): return [] original_discover_method = EndpointDiscovery.discover @@ -66,7 +66,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase): @mock.patch('keystoneclient.v2_0.client.Client') def test_discover_error_response(self, client_mock): - def discover_mock(self, type): + def discover_mock(self, manager, param=None): return ["barbican_url"] original_discover_method = EndpointDiscovery.discover @@ -94,7 +94,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase): @mock.patch('keystoneclient.v2_0.client.Client') def test_discover_response_ok(self, client_mock): - def discover_mock(self, type): + def discover_mock(self, manager, param=None): return ["barbican_url"] original_discover_method = EndpointDiscovery.discover diff --git a/doc/source/admin/telemetry-dynamic-pollster.rst b/doc/source/admin/telemetry-dynamic-pollster.rst index 0ea50acef1..f7569fa0c4 100644 --- a/doc/source/admin/telemetry-dynamic-pollster.rst +++ b/doc/source/admin/telemetry-dynamic-pollster.rst @@ -16,10 +16,6 @@ Current limitations of the dynamic pollster system Currently, the following types of APIs are not supported by the dynamic pollster system: -* Paging APIs: if a user configures a dynamic pollster to gather data - from a paging API, the pollster will use only the entries from the first - page. - * Tenant APIs: Tenant APIs are the ones that need to be polled in a tenant fashion. This feature is "a nice" to have, but is currently not implemented. @@ -583,3 +579,139 @@ are presented as follows: project_id_attribute: "user | value.split('$') | value[0]" resource_id_attribute: "user | value.split('$') | value[0]" response_entries_key: "summary" + +Handling linked API responses +----------------------------- +If the consumed API returns a linked response which contains a link to the next +response set (page), the Dynamic pollsters can be configured to follow these +links and join all linked responses into a single one. + +To enable this behavior the operator will need to configure the parameter +`next_sample_url_attribute` that must contain a mapper to the response +attribute that contains the link to the next response page. This parameter also +supports operations like the others `*_attribute` dynamic pollster's +parameters. + +Examples on how to create a pollster to handle linked API responses are +presented as follows: + +- Example of a simple linked response: + + - API response: + + .. code-block:: json + + { + "server_link": "http://test.com/v1/test-volumes/marker=c3", + "servers": [ + { + "volume": [ + { + "name": "a", + "tmp": "ra" + } + ], + "id": 1, + "name": "a1" + }, + { + "volume": [ + { + "name": "b", + "tmp": "rb" + } + ], + "id": 2, + "name": "b2" + }, + { + "volume": [ + { + "name": "c", + "tmp": "rc" + } + ], + "id": 3, + "name": "c3" + } + ] + } + + - Pollster configuration: + + .. code-block:: yaml + + --- + + - name: "dynamic.linked.response" + sample_type: "gauge" + unit: "request" + value_attribute: "[volume].tmp" + url_path: "v1/test-volumes" + response_entries_key: "servers" + next_sample_url_attribute: "server_link" + +- Example of a complex linked response: + + - API response: + + .. code-block:: json + + { + "server_link": [ + { + "href": "http://test.com/v1/test-volumes/marker=c3", + "rel": "next" + }, + { + "href": "http://test.com/v1/test-volumes/marker=b1", + "rel": "prev" + } + ], + "servers": [ + { + "volume": [ + { + "name": "a", + "tmp": "ra" + } + ], + "id": 1, + "name": "a1" + }, + { + "volume": [ + { + "name": "b", + "tmp": "rb" + } + ], + "id": 2, + "name": "b2" + }, + { + "volume": [ + { + "name": "c", + "tmp": "rc" + } + ], + "id": 3, + "name": "c3" + } + ] + } + + - Pollster configuration: + + .. code-block:: yaml + + --- + + - name: "dynamic.linked.response" + sample_type: "gauge" + unit: "request" + value_attribute: "[volume].tmp" + url_path: "v1/test-volumes" + response_entries_key: "servers" + next_sample_url_attribute: "server_link | filter(lambda v: v.get('rel') == 'next', value) | list(value) | value[0] | value.get('href')"