Browse Source

Merge "Add support to linked samples responses"

changes/77/731777/1
Zuul 1 month ago
committed by Gerrit Code Review
parent
commit
2dcd15a466
4 changed files with 187 additions and 16 deletions
  1. +34
    -5
      ceilometer/polling/dynamic_pollster.py
  2. +14
    -4
      ceilometer/tests/unit/polling/test_dynamic_pollster.py
  3. +3
    -3
      ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py
  4. +136
    -4
      doc/source/admin/telemetry-dynamic-pollster.rst

+ 34
- 5
ceilometer/polling/dynamic_pollster.py View File

@@ -361,7 +361,8 @@ class PollsterDefinitions(object):
PollsterDefinition(name='default_value', default=-1),
PollsterDefinition(name='metadata_mapping', default={}),
PollsterDefinition(name='preserve_mapped_metadata', default=True),
PollsterDefinition(name='response_entries_key')]
PollsterDefinition(name='response_entries_key'),
PollsterDefinition(name='next_sample_url_attribute')]

extra_definitions = []

@@ -481,19 +482,47 @@ class PollsterSampleGatherer(object):
response_json, url, self.definitions.configurations['name'])

if entry_size > 0:
return self.retrieve_entries_from_response(response_json)
response = self.retrieve_entries_from_response(response_json)
url_to_next_sample = self.get_url_to_next_sample(response_json)
if url_to_next_sample:
kwargs['next_sample_url'] = url_to_next_sample
response += self.execute_request_get_samples(**kwargs)
return response
return []

def get_url_to_next_sample(self, resp):
linked_sample_extractor = self.definitions.configurations[
'next_sample_url_attribute']
if not linked_sample_extractor:
return None

try:
return self.definitions.sample_extractor.\
retrieve_attribute_nested_value(resp, linked_sample_extractor)
except KeyError:
LOG.debug("There is no next sample url for the sample [%s] using "
"the configuration [%s]", resp, linked_sample_extractor)
return None

def internal_execute_request_get_samples(self, kwargs):
keystone_client = kwargs['keystone_client']
endpoint = kwargs['resource']
url = url_parse.urljoin(
endpoint, self.definitions.configurations['url_path'])
url = self.get_request_linked_samples_url(kwargs)
resp = keystone_client.session.get(url, authenticated=True)
if resp.status_code != requests.codes.ok:
resp.raise_for_status()
return resp, url

def get_request_linked_samples_url(self, kwargs):
next_sample_url = kwargs.get('next_sample_url')
if next_sample_url:
return next_sample_url
return self.get_request_url(kwargs)

def get_request_url(self, kwargs):
endpoint = kwargs['resource']
return url_parse.urljoin(
endpoint, self.definitions.configurations['url_path'])

def retrieve_entries_from_response(self, response_json):
if isinstance(response_json, list):
return response_json


+ 14
- 4
ceilometer/tests/unit/polling/test_dynamic_pollster.py View File

@@ -75,7 +75,7 @@ class PagedSamplesGenerator(SampleGenerator):
for page_link, page_size in page_links.items():
page_link = page_base_link + "/" + page_link
self.response[current_page_link] = {
self.page_link_name: page_link,
self.page_link_name: [{'href': page_link, 'rel': 'next'}],
self.dict_name: self.populate_page(page_size)
}
current_page_link = page_link
@@ -155,7 +155,7 @@ class TestDynamicPollster(base.BaseTestCase):
self.assertEqual(pollster_definition, pollster.pollster_definitions)

@mock.patch('keystoneclient.v2_0.client.Client')
def test_skip_samples(self, keystone_mock):
def test_skip_samples_with_linked_samples(self, keystone_mock):
generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={
'volume': SampleGenerator(samples_dict={
'name': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'],
@@ -179,16 +179,26 @@ class TestDynamicPollster(base.BaseTestCase):
pollster_definition['skip_sample_values'] = ['rb']
pollster_definition['url_path'] = 'v1/test-volumes'
pollster_definition['response_entries_key'] = 'servers'
pollster_definition['next_sample_url_attribute'] = \
'server_link | filter(lambda v: v.get("rel") == "next", value) |' \
'list(value) | value[0] | value.get("href")'
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
self.assertEqual(['ra', 'rc'], list(map(lambda s: s.volume, samples)))
self.assertEqual(['ra', 'rc', 'rd', 're', 'rf', 'rg', 'rh'],
list(map(lambda s: s.volume, samples)))

generator.generate_samples('http://test.com/v1/test-volumes', {
'marker=c3': 3,
'marker=f6': 3
}, 2)

pollster_definition['name'] = 'test-pollster'
pollster_definition['value_attribute'] = 'name'
pollster_definition['skip_sample_values'] = ['b2']
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
self.assertEqual(['a1', 'c3'], list(map(lambda s: s.volume, samples)))
self.assertEqual(['a1', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8'],
list(map(lambda s: s.volume, samples)))

def test_all_required_fields_ok(self):
pollster = dynamic_pollster.DynamicPollster(


+ 3
- 3
ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py View File

@@ -53,7 +53,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):
self.assertEqual(['No secrets found'], result)

def test_discover_no_barbican_endpoint(self):
def discover_mock(self, type):
def discover_mock(self, manager, param=None):
return []

original_discover_method = EndpointDiscovery.discover
@@ -67,7 +67,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):

@mock.patch('keystoneclient.v2_0.client.Client')
def test_discover_error_response(self, client_mock):
def discover_mock(self, type):
def discover_mock(self, manager, param=None):
return ["barbican_url"]

original_discover_method = EndpointDiscovery.discover
@@ -95,7 +95,7 @@ class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):

@mock.patch('keystoneclient.v2_0.client.Client')
def test_discover_response_ok(self, client_mock):
def discover_mock(self, type):
def discover_mock(self, manager, param=None):
return ["barbican_url"]

original_discover_method = EndpointDiscovery.discover


+ 136
- 4
doc/source/admin/telemetry-dynamic-pollster.rst View File

@@ -16,10 +16,6 @@ Current limitations of the dynamic pollster system
Currently, the following types of APIs are not supported by the
dynamic pollster system:

* Paging APIs: if a user configures a dynamic pollster to gather data
from a paging API, the pollster will use only the entries from the first
page.

* Tenant APIs: Tenant APIs are the ones that need to be polled in a tenant
fashion. This feature is "a nice" to have, but is currently not
implemented.
@@ -583,3 +579,139 @@ are presented as follows:
project_id_attribute: "user | value.split('$') | value[0]"
resource_id_attribute: "user | value.split('$') | value[0]"
response_entries_key: "summary"

Handling linked API responses
-----------------------------
If the consumed API returns a linked response which contains a link to the next
response set (page), the Dynamic pollsters can be configured to follow these
links and join all linked responses into a single one.

To enable this behavior the operator will need to configure the parameter
`next_sample_url_attribute` that must contain a mapper to the response
attribute that contains the link to the next response page. This parameter also
supports operations like the others `*_attribute` dynamic pollster's
parameters.

Examples on how to create a pollster to handle linked API responses are
presented as follows:

- Example of a simple linked response:

- API response:

.. code-block:: json

{
"server_link": "http://test.com/v1/test-volumes/marker=c3",
"servers": [
{
"volume": [
{
"name": "a",
"tmp": "ra"
}
],
"id": 1,
"name": "a1"
},
{
"volume": [
{
"name": "b",
"tmp": "rb"
}
],
"id": 2,
"name": "b2"
},
{
"volume": [
{
"name": "c",
"tmp": "rc"
}
],
"id": 3,
"name": "c3"
}
]
}

- Pollster configuration:

.. code-block:: yaml

---

- name: "dynamic.linked.response"
sample_type: "gauge"
unit: "request"
value_attribute: "[volume].tmp"
url_path: "v1/test-volumes"
response_entries_key: "servers"
next_sample_url_attribute: "server_link"

- Example of a complex linked response:

- API response:

.. code-block:: json

{
"server_link": [
{
"href": "http://test.com/v1/test-volumes/marker=c3",
"rel": "next"
},
{
"href": "http://test.com/v1/test-volumes/marker=b1",
"rel": "prev"
}
],
"servers": [
{
"volume": [
{
"name": "a",
"tmp": "ra"
}
],
"id": 1,
"name": "a1"
},
{
"volume": [
{
"name": "b",
"tmp": "rb"
}
],
"id": 2,
"name": "b2"
},
{
"volume": [
{
"name": "c",
"tmp": "rc"
}
],
"id": 3,
"name": "c3"
}
]
}

- Pollster configuration:

.. code-block:: yaml

---

- name: "dynamic.linked.response"
sample_type: "gauge"
unit: "request"
value_attribute: "[volume].tmp"
url_path: "v1/test-volumes"
response_entries_key: "servers"
next_sample_url_attribute: "server_link | filter(lambda v: v.get('rel') == 'next', value) | list(value) | value[0] | value.get('href')"

Loading…
Cancel
Save