Merge "Add support to linked samples responses"
This commit is contained in:
Zuul
Gerrit Code Review
commit
2dcd15a466
|
|
@ -361,7 +361,8 @@ class PollsterDefinitions(object):
|
|||
PollsterDefinition(name='default_value', default=-1),
|
||||
PollsterDefinition(name='metadata_mapping', default={}),
|
||||
PollsterDefinition(name='preserve_mapped_metadata', default=True),
|
||||
PollsterDefinition(name='response_entries_key')]
|
||||
PollsterDefinition(name='response_entries_key'),
|
||||
PollsterDefinition(name='next_sample_url_attribute')]
|
||||
|
||||
extra_definitions = []
|
||||
|
||||
|
|
@ -481,19 +482,47 @@ class PollsterSampleGatherer(object):
|
|||
response_json, url, self.definitions.configurations['name'])
|
||||
|
||||
if entry_size > 0:
|
||||
return self.retrieve_entries_from_response(response_json)
|
||||
response = self.retrieve_entries_from_response(response_json)
|
||||
url_to_next_sample = self.get_url_to_next_sample(response_json)
|
||||
if url_to_next_sample:
|
||||
kwargs['next_sample_url'] = url_to_next_sample
|
||||
response += self.execute_request_get_samples(**kwargs)
|
||||
return response
|
||||
return []
|
||||
|
||||
def get_url_to_next_sample(self, resp):
|
||||
linked_sample_extractor = self.definitions.configurations[
|
||||
'next_sample_url_attribute']
|
||||
if not linked_sample_extractor:
|
||||
return None
|
||||
|
||||
try:
|
||||
return self.definitions.sample_extractor.\
|
||||
retrieve_attribute_nested_value(resp, linked_sample_extractor)
|
||||
except KeyError:
|
||||
LOG.debug("There is no next sample url for the sample [%s] using "
|
||||
"the configuration [%s]", resp, linked_sample_extractor)
|
||||
return None
|
||||
|
||||
def internal_execute_request_get_samples(self, kwargs):
|
||||
keystone_client = kwargs['keystone_client']
|
||||
endpoint = kwargs['resource']
|
||||
url = url_parse.urljoin(
|
||||
endpoint, self.definitions.configurations['url_path'])
|
||||
url = self.get_request_linked_samples_url(kwargs)
|
||||
resp = keystone_client.session.get(url, authenticated=True)
|
||||
if resp.status_code != requests.codes.ok:
|
||||
resp.raise_for_status()
|
||||
return resp, url
|
||||
|
||||
def get_request_linked_samples_url(self, kwargs):
|
||||
next_sample_url = kwargs.get('next_sample_url')
|
||||
if next_sample_url:
|
||||
return next_sample_url
|
||||
return self.get_request_url(kwargs)
|
||||
|
||||
def get_request_url(self, kwargs):
|
||||
endpoint = kwargs['resource']
|
||||
return url_parse.urljoin(
|
||||
endpoint, self.definitions.configurations['url_path'])
|
||||
|
||||
def retrieve_entries_from_response(self, response_json):
|
||||
if isinstance(response_json, list):
|
||||
return response_json
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ class PagedSamplesGenerator(SampleGenerator):
|
|||
for page_link, page_size in page_links.items():
|
||||
page_link = page_base_link + "/" + page_link
|
||||
self.response[current_page_link] = {
|
||||
self.page_link_name: page_link,
|
||||
self.page_link_name: [{'href': page_link, 'rel': 'next'}],
|
||||
self.dict_name: self.populate_page(page_size)
|
||||
}
|
||||
current_page_link = page_link
|
||||
|
|
@ -155,7 +155,7 @@ class TestDynamicPollster(base.BaseTestCase):
|
|||
self.assertEqual(pollster_definition, pollster.pollster_definitions)
|
||||
|
||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||
def test_skip_samples(self, keystone_mock):
|
||||
def test_skip_samples_with_linked_samples(self, keystone_mock):
|
||||
generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={
|
||||
'volume': SampleGenerator(samples_dict={
|
||||
'name': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'],
|
||||
|
|
@ -179,16 +179,26 @@ class TestDynamicPollster(base.BaseTestCase):
|
|||
pollster_definition['skip_sample_values'] = ['rb']
|
||||
pollster_definition['url_path'] = 'v1/test-volumes'
|
||||
pollster_definition['response_entries_key'] = 'servers'
|
||||
pollster_definition['next_sample_url_attribute'] = \
|
||||
'server_link | filter(lambda v: v.get("rel") == "next", value) |' \
|
||||
'list(value) | value[0] | value.get("href")'
|
||||
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
|
||||
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
|
||||
self.assertEqual(['ra', 'rc'], list(map(lambda s: s.volume, samples)))
|
||||
self.assertEqual(['ra', 'rc', 'rd', 're', 'rf', 'rg', 'rh'],
|
||||
list(map(lambda s: s.volume, samples)))
|
||||
|
||||
generator.generate_samples('http://test.com/v1/test-volumes', {
|
||||
'marker=c3': 3,
|
||||
'marker=f6': 3
|
||||
}, 2)
|
||||
|
||||
pollster_definition['name'] = 'test-pollster'
|
||||
pollster_definition['value_attribute'] = 'name'
|
||||
pollster_definition['skip_sample_values'] = ['b2']
|
||||
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
|
||||
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
|
||||
self.assertEqual(['a1', 'c3'], list(map(lambda s: s.volume, samples)))
|
||||
self.assertEqual(['a1', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8'],
|
||||
list(map(lambda s: s.volume, samples)))
|
||||
|
||||
def test_all_required_fields_ok(self):
|
||||
pollster = dynamic_pollster.DynamicPollster(
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):
|
|||
self.assertEqual(['No secrets found'], result)
|
||||
|
||||
def test_discover_no_barbican_endpoint(self):
|
||||
def discover_mock(self, type):
|
||||
def discover_mock(self, manager, param=None):
|
||||
return []
|
||||
|
||||
original_discover_method = EndpointDiscovery.discover
|
||||
|
|
@ -67,7 +67,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):
|
|||
|
||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||
def test_discover_error_response(self, client_mock):
|
||||
def discover_mock(self, type):
|
||||
def discover_mock(self, manager, param=None):
|
||||
return ["barbican_url"]
|
||||
|
||||
original_discover_method = EndpointDiscovery.discover
|
||||
|
|
@ -95,7 +95,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):
|
|||
|
||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||
def test_discover_response_ok(self, client_mock):
|
||||
def discover_mock(self, type):
|
||||
def discover_mock(self, manager, param=None):
|
||||
return ["barbican_url"]
|
||||
|
||||
original_discover_method = EndpointDiscovery.discover
|
||||
|
|
|
|||
|
|
@ -16,10 +16,6 @@ Current limitations of the dynamic pollster system
|
|||
Currently, the following types of APIs are not supported by the
|
||||
dynamic pollster system:
|
||||
|
||||
* Paging APIs: if a user configures a dynamic pollster to gather data
|
||||
from a paging API, the pollster will use only the entries from the first
|
||||
page.
|
||||
|
||||
* Tenant APIs: Tenant APIs are the ones that need to be polled in a tenant
|
||||
fashion. This feature is "a nice" to have, but is currently not
|
||||
implemented.
|
||||
|
|
@ -583,3 +579,139 @@ are presented as follows:
|
|||
project_id_attribute: "user | value.split('$') | value[0]"
|
||||
resource_id_attribute: "user | value.split('$') | value[0]"
|
||||
response_entries_key: "summary"
|
||||
|
||||
Handling linked API responses
|
||||
-----------------------------
|
||||
If the consumed API returns a linked response which contains a link to the next
|
||||
response set (page), the Dynamic pollsters can be configured to follow these
|
||||
links and join all linked responses into a single one.
|
||||
|
||||
To enable this behavior the operator will need to configure the parameter
|
||||
`next_sample_url_attribute` that must contain a mapper to the response
|
||||
attribute that contains the link to the next response page. This parameter also
|
||||
supports operations like the others `*_attribute` dynamic pollster's
|
||||
parameters.
|
||||
|
||||
Examples on how to create a pollster to handle linked API responses are
|
||||
presented as follows:
|
||||
|
||||
- Example of a simple linked response:
|
||||
|
||||
- API response:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{
|
||||
"server_link": "http://test.com/v1/test-volumes/marker=c3",
|
||||
"servers": [
|
||||
{
|
||||
"volume": [
|
||||
{
|
||||
"name": "a",
|
||||
"tmp": "ra"
|
||||
}
|
||||
],
|
||||
"id": 1,
|
||||
"name": "a1"
|
||||
},
|
||||
{
|
||||
"volume": [
|
||||
{
|
||||
"name": "b",
|
||||
"tmp": "rb"
|
||||
}
|
||||
],
|
||||
"id": 2,
|
||||
"name": "b2"
|
||||
},
|
||||
{
|
||||
"volume": [
|
||||
{
|
||||
"name": "c",
|
||||
"tmp": "rc"
|
||||
}
|
||||
],
|
||||
"id": 3,
|
||||
"name": "c3"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
- Pollster configuration:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
---
|
||||
|
||||
- name: "dynamic.linked.response"
|
||||
sample_type: "gauge"
|
||||
unit: "request"
|
||||
value_attribute: "[volume].tmp"
|
||||
url_path: "v1/test-volumes"
|
||||
response_entries_key: "servers"
|
||||
next_sample_url_attribute: "server_link"
|
||||
|
||||
- Example of a complex linked response:
|
||||
|
||||
- API response:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{
|
||||
"server_link": [
|
||||
{
|
||||
"href": "http://test.com/v1/test-volumes/marker=c3",
|
||||
"rel": "next"
|
||||
},
|
||||
{
|
||||
"href": "http://test.com/v1/test-volumes/marker=b1",
|
||||
"rel": "prev"
|
||||
}
|
||||
],
|
||||
"servers": [
|
||||
{
|
||||
"volume": [
|
||||
{
|
||||
"name": "a",
|
||||
"tmp": "ra"
|
||||
}
|
||||
],
|
||||
"id": 1,
|
||||
"name": "a1"
|
||||
},
|
||||
{
|
||||
"volume": [
|
||||
{
|
||||
"name": "b",
|
||||
"tmp": "rb"
|
||||
}
|
||||
],
|
||||
"id": 2,
|
||||
"name": "b2"
|
||||
},
|
||||
{
|
||||
"volume": [
|
||||
{
|
||||
"name": "c",
|
||||
"tmp": "rc"
|
||||
}
|
||||
],
|
||||
"id": 3,
|
||||
"name": "c3"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
- Pollster configuration:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
---
|
||||
|
||||
- name: "dynamic.linked.response"
|
||||
sample_type: "gauge"
|
||||
unit: "request"
|
||||
value_attribute: "[volume].tmp"
|
||||
url_path: "v1/test-volumes"
|
||||
response_entries_key: "servers"
|
||||
next_sample_url_attribute: "server_link | filter(lambda v: v.get('rel') == 'next', value) | list(value) | value[0] | value.get('href')"
|
||||
|
|
|
|||
Loading…
Reference in New Issue