diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py index cd2c0d199f..18a39d122c 100644 --- a/ceilometer/polling/dynamic_pollster.py +++ b/ceilometer/polling/dynamic_pollster.py @@ -361,7 +361,8 @@ class PollsterDefinitions(object): PollsterDefinition(name='default_value', default=-1), PollsterDefinition(name='metadata_mapping', default={}), PollsterDefinition(name='preserve_mapped_metadata', default=True), - PollsterDefinition(name='response_entries_key')] + PollsterDefinition(name='response_entries_key'), + PollsterDefinition(name='next_sample_url_attribute')] extra_definitions = [] @@ -481,19 +482,47 @@ class PollsterSampleGatherer(object): response_json, url, self.definitions.configurations['name']) if entry_size > 0: - return self.retrieve_entries_from_response(response_json) + response = self.retrieve_entries_from_response(response_json) + url_to_next_sample = self.get_url_to_next_sample(response_json) + if url_to_next_sample: + kwargs['next_sample_url'] = url_to_next_sample + response += self.execute_request_get_samples(**kwargs) + return response return [] + def get_url_to_next_sample(self, resp): + linked_sample_extractor = self.definitions.configurations[ + 'next_sample_url_attribute'] + if not linked_sample_extractor: + return None + + try: + return self.definitions.sample_extractor.\ + retrieve_attribute_nested_value(resp, linked_sample_extractor) + except KeyError: + LOG.debug("There is no next sample url for the sample [%s] using " + "the configuration [%s]", resp, linked_sample_extractor) + return None + def internal_execute_request_get_samples(self, kwargs): keystone_client = kwargs['keystone_client'] - endpoint = kwargs['resource'] - url = url_parse.urljoin( - endpoint, self.definitions.configurations['url_path']) + url = self.get_request_linked_samples_url(kwargs) resp = keystone_client.session.get(url, authenticated=True) if resp.status_code != requests.codes.ok: resp.raise_for_status() return resp, url + def get_request_linked_samples_url(self, kwargs): + next_sample_url = kwargs.get('next_sample_url') + if next_sample_url: + return next_sample_url + return self.get_request_url(kwargs) + + def get_request_url(self, kwargs): + endpoint = kwargs['resource'] + return url_parse.urljoin( + endpoint, self.definitions.configurations['url_path']) + def retrieve_entries_from_response(self, response_json): if isinstance(response_json, list): return response_json diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py index 53b65cc6c9..a9c7bc016a 100644 --- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py @@ -75,7 +75,7 @@ class PagedSamplesGenerator(SampleGenerator): for page_link, page_size in page_links.items(): page_link = page_base_link + "/" + page_link self.response[current_page_link] = { - self.page_link_name: page_link, + self.page_link_name: [{'href': page_link, 'rel': 'next'}], self.dict_name: self.populate_page(page_size) } current_page_link = page_link @@ -155,7 +155,7 @@ class TestDynamicPollster(base.BaseTestCase): self.assertEqual(pollster_definition, pollster.pollster_definitions) @mock.patch('keystoneclient.v2_0.client.Client') - def test_skip_samples(self, keystone_mock): + def test_skip_samples_with_linked_samples(self, keystone_mock): generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={ 'volume': SampleGenerator(samples_dict={ 'name': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'], @@ -179,16 +179,26 @@ class TestDynamicPollster(base.BaseTestCase): pollster_definition['skip_sample_values'] = ['rb'] pollster_definition['url_path'] = 'v1/test-volumes' pollster_definition['response_entries_key'] = 'servers' + pollster_definition['next_sample_url_attribute'] = \ + 'server_link | filter(lambda v: v.get("rel") == "next", value) |' \ + 'list(value) | value[0] | value.get("href")' pollster = dynamic_pollster.DynamicPollster(pollster_definition) samples = pollster.get_samples(fake_manager, None, ['http://test.com']) - self.assertEqual(['ra', 'rc'], list(map(lambda s: s.volume, samples))) + self.assertEqual(['ra', 'rc', 'rd', 're', 'rf', 'rg', 'rh'], + list(map(lambda s: s.volume, samples))) + + generator.generate_samples('http://test.com/v1/test-volumes', { + 'marker=c3': 3, + 'marker=f6': 3 + }, 2) pollster_definition['name'] = 'test-pollster' pollster_definition['value_attribute'] = 'name' pollster_definition['skip_sample_values'] = ['b2'] pollster = dynamic_pollster.DynamicPollster(pollster_definition) samples = pollster.get_samples(fake_manager, None, ['http://test.com']) - self.assertEqual(['a1', 'c3'], list(map(lambda s: s.volume, samples))) + self.assertEqual(['a1', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8'], + list(map(lambda s: s.volume, samples))) def test_all_required_fields_ok(self): pollster = dynamic_pollster.DynamicPollster( diff --git a/ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py b/ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py index b1bb4c3dd0..c1fffd87b5 100644 --- a/ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py +++ b/ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py @@ -53,7 +53,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase): self.assertEqual(['No secrets found'], result) def test_discover_no_barbican_endpoint(self): - def discover_mock(self, type): + def discover_mock(self, manager, param=None): return [] original_discover_method = EndpointDiscovery.discover @@ -67,7 +67,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase): @mock.patch('keystoneclient.v2_0.client.Client') def test_discover_error_response(self, client_mock): - def discover_mock(self, type): + def discover_mock(self, manager, param=None): return ["barbican_url"] original_discover_method = EndpointDiscovery.discover @@ -95,7 +95,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase): @mock.patch('keystoneclient.v2_0.client.Client') def test_discover_response_ok(self, client_mock): - def discover_mock(self, type): + def discover_mock(self, manager, param=None): return ["barbican_url"] original_discover_method = EndpointDiscovery.discover diff --git a/doc/source/admin/telemetry-dynamic-pollster.rst b/doc/source/admin/telemetry-dynamic-pollster.rst index 0ea50acef1..f7569fa0c4 100644 --- a/doc/source/admin/telemetry-dynamic-pollster.rst +++ b/doc/source/admin/telemetry-dynamic-pollster.rst @@ -16,10 +16,6 @@ Current limitations of the dynamic pollster system Currently, the following types of APIs are not supported by the dynamic pollster system: -* Paging APIs: if a user configures a dynamic pollster to gather data - from a paging API, the pollster will use only the entries from the first - page. - * Tenant APIs: Tenant APIs are the ones that need to be polled in a tenant fashion. This feature is "a nice" to have, but is currently not implemented. @@ -583,3 +579,139 @@ are presented as follows: project_id_attribute: "user | value.split('$') | value[0]" resource_id_attribute: "user | value.split('$') | value[0]" response_entries_key: "summary" + +Handling linked API responses +----------------------------- +If the consumed API returns a linked response which contains a link to the next +response set (page), the Dynamic pollsters can be configured to follow these +links and join all linked responses into a single one. + +To enable this behavior the operator will need to configure the parameter +`next_sample_url_attribute` that must contain a mapper to the response +attribute that contains the link to the next response page. This parameter also +supports operations like the others `*_attribute` dynamic pollster's +parameters. + +Examples on how to create a pollster to handle linked API responses are +presented as follows: + +- Example of a simple linked response: + + - API response: + + .. code-block:: json + + { + "server_link": "http://test.com/v1/test-volumes/marker=c3", + "servers": [ + { + "volume": [ + { + "name": "a", + "tmp": "ra" + } + ], + "id": 1, + "name": "a1" + }, + { + "volume": [ + { + "name": "b", + "tmp": "rb" + } + ], + "id": 2, + "name": "b2" + }, + { + "volume": [ + { + "name": "c", + "tmp": "rc" + } + ], + "id": 3, + "name": "c3" + } + ] + } + + - Pollster configuration: + + .. code-block:: yaml + + --- + + - name: "dynamic.linked.response" + sample_type: "gauge" + unit: "request" + value_attribute: "[volume].tmp" + url_path: "v1/test-volumes" + response_entries_key: "servers" + next_sample_url_attribute: "server_link" + +- Example of a complex linked response: + + - API response: + + .. code-block:: json + + { + "server_link": [ + { + "href": "http://test.com/v1/test-volumes/marker=c3", + "rel": "next" + }, + { + "href": "http://test.com/v1/test-volumes/marker=b1", + "rel": "prev" + } + ], + "servers": [ + { + "volume": [ + { + "name": "a", + "tmp": "ra" + } + ], + "id": 1, + "name": "a1" + }, + { + "volume": [ + { + "name": "b", + "tmp": "rb" + } + ], + "id": 2, + "name": "b2" + }, + { + "volume": [ + { + "name": "c", + "tmp": "rc" + } + ], + "id": 3, + "name": "c3" + } + ] + } + + - Pollster configuration: + + .. code-block:: yaml + + --- + + - name: "dynamic.linked.response" + sample_type: "gauge" + unit: "request" + value_attribute: "[volume].tmp" + url_path: "v1/test-volumes" + response_entries_key: "servers" + next_sample_url_attribute: "server_link | filter(lambda v: v.get('rel') == 'next', value) | list(value) | value[0] | value.get('href')"