Merge "Add support to linked samples responses"
This commit is contained in:
commit
2dcd15a466
|
@ -361,7 +361,8 @@ class PollsterDefinitions(object):
|
||||||
PollsterDefinition(name='default_value', default=-1),
|
PollsterDefinition(name='default_value', default=-1),
|
||||||
PollsterDefinition(name='metadata_mapping', default={}),
|
PollsterDefinition(name='metadata_mapping', default={}),
|
||||||
PollsterDefinition(name='preserve_mapped_metadata', default=True),
|
PollsterDefinition(name='preserve_mapped_metadata', default=True),
|
||||||
PollsterDefinition(name='response_entries_key')]
|
PollsterDefinition(name='response_entries_key'),
|
||||||
|
PollsterDefinition(name='next_sample_url_attribute')]
|
||||||
|
|
||||||
extra_definitions = []
|
extra_definitions = []
|
||||||
|
|
||||||
|
@ -481,19 +482,47 @@ class PollsterSampleGatherer(object):
|
||||||
response_json, url, self.definitions.configurations['name'])
|
response_json, url, self.definitions.configurations['name'])
|
||||||
|
|
||||||
if entry_size > 0:
|
if entry_size > 0:
|
||||||
return self.retrieve_entries_from_response(response_json)
|
response = self.retrieve_entries_from_response(response_json)
|
||||||
|
url_to_next_sample = self.get_url_to_next_sample(response_json)
|
||||||
|
if url_to_next_sample:
|
||||||
|
kwargs['next_sample_url'] = url_to_next_sample
|
||||||
|
response += self.execute_request_get_samples(**kwargs)
|
||||||
|
return response
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
def get_url_to_next_sample(self, resp):
|
||||||
|
linked_sample_extractor = self.definitions.configurations[
|
||||||
|
'next_sample_url_attribute']
|
||||||
|
if not linked_sample_extractor:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
return self.definitions.sample_extractor.\
|
||||||
|
retrieve_attribute_nested_value(resp, linked_sample_extractor)
|
||||||
|
except KeyError:
|
||||||
|
LOG.debug("There is no next sample url for the sample [%s] using "
|
||||||
|
"the configuration [%s]", resp, linked_sample_extractor)
|
||||||
|
return None
|
||||||
|
|
||||||
def internal_execute_request_get_samples(self, kwargs):
|
def internal_execute_request_get_samples(self, kwargs):
|
||||||
keystone_client = kwargs['keystone_client']
|
keystone_client = kwargs['keystone_client']
|
||||||
endpoint = kwargs['resource']
|
url = self.get_request_linked_samples_url(kwargs)
|
||||||
url = url_parse.urljoin(
|
|
||||||
endpoint, self.definitions.configurations['url_path'])
|
|
||||||
resp = keystone_client.session.get(url, authenticated=True)
|
resp = keystone_client.session.get(url, authenticated=True)
|
||||||
if resp.status_code != requests.codes.ok:
|
if resp.status_code != requests.codes.ok:
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
return resp, url
|
return resp, url
|
||||||
|
|
||||||
|
def get_request_linked_samples_url(self, kwargs):
|
||||||
|
next_sample_url = kwargs.get('next_sample_url')
|
||||||
|
if next_sample_url:
|
||||||
|
return next_sample_url
|
||||||
|
return self.get_request_url(kwargs)
|
||||||
|
|
||||||
|
def get_request_url(self, kwargs):
|
||||||
|
endpoint = kwargs['resource']
|
||||||
|
return url_parse.urljoin(
|
||||||
|
endpoint, self.definitions.configurations['url_path'])
|
||||||
|
|
||||||
def retrieve_entries_from_response(self, response_json):
|
def retrieve_entries_from_response(self, response_json):
|
||||||
if isinstance(response_json, list):
|
if isinstance(response_json, list):
|
||||||
return response_json
|
return response_json
|
||||||
|
|
|
@ -75,7 +75,7 @@ class PagedSamplesGenerator(SampleGenerator):
|
||||||
for page_link, page_size in page_links.items():
|
for page_link, page_size in page_links.items():
|
||||||
page_link = page_base_link + "/" + page_link
|
page_link = page_base_link + "/" + page_link
|
||||||
self.response[current_page_link] = {
|
self.response[current_page_link] = {
|
||||||
self.page_link_name: page_link,
|
self.page_link_name: [{'href': page_link, 'rel': 'next'}],
|
||||||
self.dict_name: self.populate_page(page_size)
|
self.dict_name: self.populate_page(page_size)
|
||||||
}
|
}
|
||||||
current_page_link = page_link
|
current_page_link = page_link
|
||||||
|
@ -155,7 +155,7 @@ class TestDynamicPollster(base.BaseTestCase):
|
||||||
self.assertEqual(pollster_definition, pollster.pollster_definitions)
|
self.assertEqual(pollster_definition, pollster.pollster_definitions)
|
||||||
|
|
||||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||||
def test_skip_samples(self, keystone_mock):
|
def test_skip_samples_with_linked_samples(self, keystone_mock):
|
||||||
generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={
|
generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={
|
||||||
'volume': SampleGenerator(samples_dict={
|
'volume': SampleGenerator(samples_dict={
|
||||||
'name': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'],
|
'name': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'],
|
||||||
|
@ -179,16 +179,26 @@ class TestDynamicPollster(base.BaseTestCase):
|
||||||
pollster_definition['skip_sample_values'] = ['rb']
|
pollster_definition['skip_sample_values'] = ['rb']
|
||||||
pollster_definition['url_path'] = 'v1/test-volumes'
|
pollster_definition['url_path'] = 'v1/test-volumes'
|
||||||
pollster_definition['response_entries_key'] = 'servers'
|
pollster_definition['response_entries_key'] = 'servers'
|
||||||
|
pollster_definition['next_sample_url_attribute'] = \
|
||||||
|
'server_link | filter(lambda v: v.get("rel") == "next", value) |' \
|
||||||
|
'list(value) | value[0] | value.get("href")'
|
||||||
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
|
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
|
||||||
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
|
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
|
||||||
self.assertEqual(['ra', 'rc'], list(map(lambda s: s.volume, samples)))
|
self.assertEqual(['ra', 'rc', 'rd', 're', 'rf', 'rg', 'rh'],
|
||||||
|
list(map(lambda s: s.volume, samples)))
|
||||||
|
|
||||||
|
generator.generate_samples('http://test.com/v1/test-volumes', {
|
||||||
|
'marker=c3': 3,
|
||||||
|
'marker=f6': 3
|
||||||
|
}, 2)
|
||||||
|
|
||||||
pollster_definition['name'] = 'test-pollster'
|
pollster_definition['name'] = 'test-pollster'
|
||||||
pollster_definition['value_attribute'] = 'name'
|
pollster_definition['value_attribute'] = 'name'
|
||||||
pollster_definition['skip_sample_values'] = ['b2']
|
pollster_definition['skip_sample_values'] = ['b2']
|
||||||
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
|
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
|
||||||
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
|
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
|
||||||
self.assertEqual(['a1', 'c3'], list(map(lambda s: s.volume, samples)))
|
self.assertEqual(['a1', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8'],
|
||||||
|
list(map(lambda s: s.volume, samples)))
|
||||||
|
|
||||||
def test_all_required_fields_ok(self):
|
def test_all_required_fields_ok(self):
|
||||||
pollster = dynamic_pollster.DynamicPollster(
|
pollster = dynamic_pollster.DynamicPollster(
|
||||||
|
|
|
@ -53,7 +53,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):
|
||||||
self.assertEqual(['No secrets found'], result)
|
self.assertEqual(['No secrets found'], result)
|
||||||
|
|
||||||
def test_discover_no_barbican_endpoint(self):
|
def test_discover_no_barbican_endpoint(self):
|
||||||
def discover_mock(self, type):
|
def discover_mock(self, manager, param=None):
|
||||||
return []
|
return []
|
||||||
|
|
||||||
original_discover_method = EndpointDiscovery.discover
|
original_discover_method = EndpointDiscovery.discover
|
||||||
|
@ -67,7 +67,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):
|
||||||
|
|
||||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||||
def test_discover_error_response(self, client_mock):
|
def test_discover_error_response(self, client_mock):
|
||||||
def discover_mock(self, type):
|
def discover_mock(self, manager, param=None):
|
||||||
return ["barbican_url"]
|
return ["barbican_url"]
|
||||||
|
|
||||||
original_discover_method = EndpointDiscovery.discover
|
original_discover_method = EndpointDiscovery.discover
|
||||||
|
@ -95,7 +95,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):
|
||||||
|
|
||||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||||
def test_discover_response_ok(self, client_mock):
|
def test_discover_response_ok(self, client_mock):
|
||||||
def discover_mock(self, type):
|
def discover_mock(self, manager, param=None):
|
||||||
return ["barbican_url"]
|
return ["barbican_url"]
|
||||||
|
|
||||||
original_discover_method = EndpointDiscovery.discover
|
original_discover_method = EndpointDiscovery.discover
|
||||||
|
|
|
@ -16,10 +16,6 @@ Current limitations of the dynamic pollster system
|
||||||
Currently, the following types of APIs are not supported by the
|
Currently, the following types of APIs are not supported by the
|
||||||
dynamic pollster system:
|
dynamic pollster system:
|
||||||
|
|
||||||
* Paging APIs: if a user configures a dynamic pollster to gather data
|
|
||||||
from a paging API, the pollster will use only the entries from the first
|
|
||||||
page.
|
|
||||||
|
|
||||||
* Tenant APIs: Tenant APIs are the ones that need to be polled in a tenant
|
* Tenant APIs: Tenant APIs are the ones that need to be polled in a tenant
|
||||||
fashion. This feature is "a nice" to have, but is currently not
|
fashion. This feature is "a nice" to have, but is currently not
|
||||||
implemented.
|
implemented.
|
||||||
|
@ -583,3 +579,139 @@ are presented as follows:
|
||||||
project_id_attribute: "user | value.split('$') | value[0]"
|
project_id_attribute: "user | value.split('$') | value[0]"
|
||||||
resource_id_attribute: "user | value.split('$') | value[0]"
|
resource_id_attribute: "user | value.split('$') | value[0]"
|
||||||
response_entries_key: "summary"
|
response_entries_key: "summary"
|
||||||
|
|
||||||
|
Handling linked API responses
|
||||||
|
-----------------------------
|
||||||
|
If the consumed API returns a linked response which contains a link to the next
|
||||||
|
response set (page), the Dynamic pollsters can be configured to follow these
|
||||||
|
links and join all linked responses into a single one.
|
||||||
|
|
||||||
|
To enable this behavior the operator will need to configure the parameter
|
||||||
|
`next_sample_url_attribute` that must contain a mapper to the response
|
||||||
|
attribute that contains the link to the next response page. This parameter also
|
||||||
|
supports operations like the others `*_attribute` dynamic pollster's
|
||||||
|
parameters.
|
||||||
|
|
||||||
|
Examples on how to create a pollster to handle linked API responses are
|
||||||
|
presented as follows:
|
||||||
|
|
||||||
|
- Example of a simple linked response:
|
||||||
|
|
||||||
|
- API response:
|
||||||
|
|
||||||
|
.. code-block:: json
|
||||||
|
|
||||||
|
{
|
||||||
|
"server_link": "http://test.com/v1/test-volumes/marker=c3",
|
||||||
|
"servers": [
|
||||||
|
{
|
||||||
|
"volume": [
|
||||||
|
{
|
||||||
|
"name": "a",
|
||||||
|
"tmp": "ra"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"id": 1,
|
||||||
|
"name": "a1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"volume": [
|
||||||
|
{
|
||||||
|
"name": "b",
|
||||||
|
"tmp": "rb"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"id": 2,
|
||||||
|
"name": "b2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"volume": [
|
||||||
|
{
|
||||||
|
"name": "c",
|
||||||
|
"tmp": "rc"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"id": 3,
|
||||||
|
"name": "c3"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
- Pollster configuration:
|
||||||
|
|
||||||
|
.. code-block:: yaml
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
- name: "dynamic.linked.response"
|
||||||
|
sample_type: "gauge"
|
||||||
|
unit: "request"
|
||||||
|
value_attribute: "[volume].tmp"
|
||||||
|
url_path: "v1/test-volumes"
|
||||||
|
response_entries_key: "servers"
|
||||||
|
next_sample_url_attribute: "server_link"
|
||||||
|
|
||||||
|
- Example of a complex linked response:
|
||||||
|
|
||||||
|
- API response:
|
||||||
|
|
||||||
|
.. code-block:: json
|
||||||
|
|
||||||
|
{
|
||||||
|
"server_link": [
|
||||||
|
{
|
||||||
|
"href": "http://test.com/v1/test-volumes/marker=c3",
|
||||||
|
"rel": "next"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "http://test.com/v1/test-volumes/marker=b1",
|
||||||
|
"rel": "prev"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"servers": [
|
||||||
|
{
|
||||||
|
"volume": [
|
||||||
|
{
|
||||||
|
"name": "a",
|
||||||
|
"tmp": "ra"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"id": 1,
|
||||||
|
"name": "a1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"volume": [
|
||||||
|
{
|
||||||
|
"name": "b",
|
||||||
|
"tmp": "rb"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"id": 2,
|
||||||
|
"name": "b2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"volume": [
|
||||||
|
{
|
||||||
|
"name": "c",
|
||||||
|
"tmp": "rc"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"id": 3,
|
||||||
|
"name": "c3"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
- Pollster configuration:
|
||||||
|
|
||||||
|
.. code-block:: yaml
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
- name: "dynamic.linked.response"
|
||||||
|
sample_type: "gauge"
|
||||||
|
unit: "request"
|
||||||
|
value_attribute: "[volume].tmp"
|
||||||
|
url_path: "v1/test-volumes"
|
||||||
|
response_entries_key: "servers"
|
||||||
|
next_sample_url_attribute: "server_link | filter(lambda v: v.get('rel') == 'next', value) | list(value) | value[0] | value.get('href')"
|
||||||
|
|
Loading…
Reference in New Issue