Merge "Add support to linked samples responses"

This commit is contained in:
Zuul 2020-05-29 12:53:45 +00:00 committed by Gerrit Code Review
commit 2dcd15a466
4 changed files with 187 additions and 16 deletions

View File

@ -361,7 +361,8 @@ class PollsterDefinitions(object):
PollsterDefinition(name='default_value', default=-1), PollsterDefinition(name='default_value', default=-1),
PollsterDefinition(name='metadata_mapping', default={}), PollsterDefinition(name='metadata_mapping', default={}),
PollsterDefinition(name='preserve_mapped_metadata', default=True), PollsterDefinition(name='preserve_mapped_metadata', default=True),
PollsterDefinition(name='response_entries_key')] PollsterDefinition(name='response_entries_key'),
PollsterDefinition(name='next_sample_url_attribute')]
extra_definitions = [] extra_definitions = []
@ -481,19 +482,47 @@ class PollsterSampleGatherer(object):
response_json, url, self.definitions.configurations['name']) response_json, url, self.definitions.configurations['name'])
if entry_size > 0: if entry_size > 0:
return self.retrieve_entries_from_response(response_json) response = self.retrieve_entries_from_response(response_json)
url_to_next_sample = self.get_url_to_next_sample(response_json)
if url_to_next_sample:
kwargs['next_sample_url'] = url_to_next_sample
response += self.execute_request_get_samples(**kwargs)
return response
return [] return []
def get_url_to_next_sample(self, resp):
linked_sample_extractor = self.definitions.configurations[
'next_sample_url_attribute']
if not linked_sample_extractor:
return None
try:
return self.definitions.sample_extractor.\
retrieve_attribute_nested_value(resp, linked_sample_extractor)
except KeyError:
LOG.debug("There is no next sample url for the sample [%s] using "
"the configuration [%s]", resp, linked_sample_extractor)
return None
def internal_execute_request_get_samples(self, kwargs): def internal_execute_request_get_samples(self, kwargs):
keystone_client = kwargs['keystone_client'] keystone_client = kwargs['keystone_client']
endpoint = kwargs['resource'] url = self.get_request_linked_samples_url(kwargs)
url = url_parse.urljoin(
endpoint, self.definitions.configurations['url_path'])
resp = keystone_client.session.get(url, authenticated=True) resp = keystone_client.session.get(url, authenticated=True)
if resp.status_code != requests.codes.ok: if resp.status_code != requests.codes.ok:
resp.raise_for_status() resp.raise_for_status()
return resp, url return resp, url
def get_request_linked_samples_url(self, kwargs):
next_sample_url = kwargs.get('next_sample_url')
if next_sample_url:
return next_sample_url
return self.get_request_url(kwargs)
def get_request_url(self, kwargs):
endpoint = kwargs['resource']
return url_parse.urljoin(
endpoint, self.definitions.configurations['url_path'])
def retrieve_entries_from_response(self, response_json): def retrieve_entries_from_response(self, response_json):
if isinstance(response_json, list): if isinstance(response_json, list):
return response_json return response_json

View File

@ -75,7 +75,7 @@ class PagedSamplesGenerator(SampleGenerator):
for page_link, page_size in page_links.items(): for page_link, page_size in page_links.items():
page_link = page_base_link + "/" + page_link page_link = page_base_link + "/" + page_link
self.response[current_page_link] = { self.response[current_page_link] = {
self.page_link_name: page_link, self.page_link_name: [{'href': page_link, 'rel': 'next'}],
self.dict_name: self.populate_page(page_size) self.dict_name: self.populate_page(page_size)
} }
current_page_link = page_link current_page_link = page_link
@ -155,7 +155,7 @@ class TestDynamicPollster(base.BaseTestCase):
self.assertEqual(pollster_definition, pollster.pollster_definitions) self.assertEqual(pollster_definition, pollster.pollster_definitions)
@mock.patch('keystoneclient.v2_0.client.Client') @mock.patch('keystoneclient.v2_0.client.Client')
def test_skip_samples(self, keystone_mock): def test_skip_samples_with_linked_samples(self, keystone_mock):
generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={ generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={
'volume': SampleGenerator(samples_dict={ 'volume': SampleGenerator(samples_dict={
'name': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'], 'name': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'],
@ -179,16 +179,26 @@ class TestDynamicPollster(base.BaseTestCase):
pollster_definition['skip_sample_values'] = ['rb'] pollster_definition['skip_sample_values'] = ['rb']
pollster_definition['url_path'] = 'v1/test-volumes' pollster_definition['url_path'] = 'v1/test-volumes'
pollster_definition['response_entries_key'] = 'servers' pollster_definition['response_entries_key'] = 'servers'
pollster_definition['next_sample_url_attribute'] = \
'server_link | filter(lambda v: v.get("rel") == "next", value) |' \
'list(value) | value[0] | value.get("href")'
pollster = dynamic_pollster.DynamicPollster(pollster_definition) pollster = dynamic_pollster.DynamicPollster(pollster_definition)
samples = pollster.get_samples(fake_manager, None, ['http://test.com']) samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
self.assertEqual(['ra', 'rc'], list(map(lambda s: s.volume, samples))) self.assertEqual(['ra', 'rc', 'rd', 're', 'rf', 'rg', 'rh'],
list(map(lambda s: s.volume, samples)))
generator.generate_samples('http://test.com/v1/test-volumes', {
'marker=c3': 3,
'marker=f6': 3
}, 2)
pollster_definition['name'] = 'test-pollster' pollster_definition['name'] = 'test-pollster'
pollster_definition['value_attribute'] = 'name' pollster_definition['value_attribute'] = 'name'
pollster_definition['skip_sample_values'] = ['b2'] pollster_definition['skip_sample_values'] = ['b2']
pollster = dynamic_pollster.DynamicPollster(pollster_definition) pollster = dynamic_pollster.DynamicPollster(pollster_definition)
samples = pollster.get_samples(fake_manager, None, ['http://test.com']) samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
self.assertEqual(['a1', 'c3'], list(map(lambda s: s.volume, samples))) self.assertEqual(['a1', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8'],
list(map(lambda s: s.volume, samples)))
def test_all_required_fields_ok(self): def test_all_required_fields_ok(self):
pollster = dynamic_pollster.DynamicPollster( pollster = dynamic_pollster.DynamicPollster(

View File

@ -53,7 +53,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):
self.assertEqual(['No secrets found'], result) self.assertEqual(['No secrets found'], result)
def test_discover_no_barbican_endpoint(self): def test_discover_no_barbican_endpoint(self):
def discover_mock(self, type): def discover_mock(self, manager, param=None):
return [] return []
original_discover_method = EndpointDiscovery.discover original_discover_method = EndpointDiscovery.discover
@ -67,7 +67,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):
@mock.patch('keystoneclient.v2_0.client.Client') @mock.patch('keystoneclient.v2_0.client.Client')
def test_discover_error_response(self, client_mock): def test_discover_error_response(self, client_mock):
def discover_mock(self, type): def discover_mock(self, manager, param=None):
return ["barbican_url"] return ["barbican_url"]
original_discover_method = EndpointDiscovery.discover original_discover_method = EndpointDiscovery.discover
@ -95,7 +95,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):
@mock.patch('keystoneclient.v2_0.client.Client') @mock.patch('keystoneclient.v2_0.client.Client')
def test_discover_response_ok(self, client_mock): def test_discover_response_ok(self, client_mock):
def discover_mock(self, type): def discover_mock(self, manager, param=None):
return ["barbican_url"] return ["barbican_url"]
original_discover_method = EndpointDiscovery.discover original_discover_method = EndpointDiscovery.discover

View File

@ -16,10 +16,6 @@ Current limitations of the dynamic pollster system
Currently, the following types of APIs are not supported by the Currently, the following types of APIs are not supported by the
dynamic pollster system: dynamic pollster system:
* Paging APIs: if a user configures a dynamic pollster to gather data
from a paging API, the pollster will use only the entries from the first
page.
* Tenant APIs: Tenant APIs are the ones that need to be polled in a tenant * Tenant APIs: Tenant APIs are the ones that need to be polled in a tenant
fashion. This feature is "a nice" to have, but is currently not fashion. This feature is "a nice" to have, but is currently not
implemented. implemented.
@ -583,3 +579,139 @@ are presented as follows:
project_id_attribute: "user | value.split('$') | value[0]" project_id_attribute: "user | value.split('$') | value[0]"
resource_id_attribute: "user | value.split('$') | value[0]" resource_id_attribute: "user | value.split('$') | value[0]"
response_entries_key: "summary" response_entries_key: "summary"
Handling linked API responses
-----------------------------
If the consumed API returns a linked response which contains a link to the next
response set (page), the Dynamic pollsters can be configured to follow these
links and join all linked responses into a single one.
To enable this behavior the operator will need to configure the parameter
`next_sample_url_attribute` that must contain a mapper to the response
attribute that contains the link to the next response page. This parameter also
supports operations like the others `*_attribute` dynamic pollster's
parameters.
Examples on how to create a pollster to handle linked API responses are
presented as follows:
- Example of a simple linked response:
- API response:
.. code-block:: json
{
"server_link": "http://test.com/v1/test-volumes/marker=c3",
"servers": [
{
"volume": [
{
"name": "a",
"tmp": "ra"
}
],
"id": 1,
"name": "a1"
},
{
"volume": [
{
"name": "b",
"tmp": "rb"
}
],
"id": 2,
"name": "b2"
},
{
"volume": [
{
"name": "c",
"tmp": "rc"
}
],
"id": 3,
"name": "c3"
}
]
}
- Pollster configuration:
.. code-block:: yaml
---
- name: "dynamic.linked.response"
sample_type: "gauge"
unit: "request"
value_attribute: "[volume].tmp"
url_path: "v1/test-volumes"
response_entries_key: "servers"
next_sample_url_attribute: "server_link"
- Example of a complex linked response:
- API response:
.. code-block:: json
{
"server_link": [
{
"href": "http://test.com/v1/test-volumes/marker=c3",
"rel": "next"
},
{
"href": "http://test.com/v1/test-volumes/marker=b1",
"rel": "prev"
}
],
"servers": [
{
"volume": [
{
"name": "a",
"tmp": "ra"
}
],
"id": 1,
"name": "a1"
},
{
"volume": [
{
"name": "b",
"tmp": "rb"
}
],
"id": 2,
"name": "b2"
},
{
"volume": [
{
"name": "c",
"tmp": "rc"
}
],
"id": 3,
"name": "c3"
}
]
}
- Pollster configuration:
.. code-block:: yaml
---
- name: "dynamic.linked.response"
sample_type: "gauge"
unit: "request"
value_attribute: "[volume].tmp"
url_path: "v1/test-volumes"
response_entries_key: "servers"
next_sample_url_attribute: "server_link | filter(lambda v: v.get('rel') == 'next', value) | list(value) | value[0] | value.get('href')"