Dynamic pollster support paging systems that return only a path for the next page

This was found when working with Glance API, where the next page attribute only
contains the context for the next call, and not a full qualified web address.
Moreover, while implementing this one, I noticed the there was no paging for
the non-openstack pollster. Therefore, I added it as well.

Depends-On: https://review.opendev.org/#/c/709807/

Change-Id: I3bc3cd4a7aa58ca1e2a57d383933493f5dc377ca
This commit is contained in:
Rafael Weingärtner 2020-05-28 21:32:11 -03:00
parent 51f06518c5
commit 9c4f0d9391
3 changed files with 117 additions and 11 deletions

View File

@ -618,13 +618,19 @@ class PollsterSampleGatherer(object):
def get_request_linked_samples_url(self, kwargs):
next_sample_url = kwargs.get('next_sample_url')
if next_sample_url:
return next_sample_url
return self.get_request_url(kwargs)
return self.get_next_page_url(kwargs, next_sample_url)
return self.get_request_url(
kwargs, self.definitions.configurations['url_path'])
def get_request_url(self, kwargs):
def get_next_page_url(self, kwargs, next_sample_url):
parse_result = url_parse.urlparse(next_sample_url)
if parse_result.netloc:
return next_sample_url
return self.get_request_url(kwargs, next_sample_url)
def get_request_url(self, kwargs, url_path):
endpoint = kwargs['resource']
return url_parse.urljoin(
endpoint, self.definitions.configurations['url_path'])
return url_parse.urljoin(endpoint, url_path)
def retrieve_entries_from_response(self, response_json):
if isinstance(response_json, list):
@ -677,7 +683,7 @@ class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer):
if override_credentials:
credentials = override_credentials
url = self.definitions.configurations['url_path']
url = self.get_request_linked_samples_url(kwargs)
authenticator_module_name = self.definitions.configurations['module']
authenticator_class_name = \
@ -714,6 +720,12 @@ class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer):
return request_arguments
def get_request_url(self, kwargs, url_path):
endpoint = self.definitions.configurations['url_path']
if endpoint == url_path:
return url_path
return url_parse.urljoin(endpoint, url_path)
def execute_request_get_samples(self, **kwargs):
samples = super(NonOpenStackApisSamplesGatherer,
self).execute_request_get_samples(**kwargs)

View File

@ -11,21 +11,22 @@
# License for the specific language governing permissions and limitations
# under the License.
"""Tests for ceilometer/polling/dynamic_pollster.py"""
"""Tests for OpenStack dynamic pollster
"""
import copy
import logging
from unittest import mock
from oslotest import base
import requests
from six.moves.urllib import parse as url_parse
from ceilometer.declarative import DynamicPollsterDefinitionException
from ceilometer.polling import dynamic_pollster
from ceilometer import sample
from oslotest import base
LOG = logging.getLogger(__name__)
REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit',
'value_attribute', 'endpoint_type',
'url_path']
@ -889,3 +890,48 @@ class TestDynamicPollster(base.BaseTestCase):
self.assertEqual(list(range(1, 9)),
list(map(lambda s: s.resource_metadata["flavor.ram"],
samples)))
def test_get_request_linked_samples_url_no_next_sample(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
base_url = "http://test.com/something_that_we_do_not_care"
expected_url = url_parse.urljoin(
base_url, self.pollster_definition_only_required_fields[
'url_path'])
kwargs = {'resource': base_url}
url = pollster.definitions.sample_gatherer\
.get_request_linked_samples_url(kwargs)
self.assertEqual(expected_url, url)
def test_get_request_linked_samples_url_next_sample_url(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
base_url = "http://test.com/something_that_we_do_not_care"
expected_url = "http://test.com/next_page"
kwargs = {'resource': base_url,
'next_sample_url': expected_url}
url = pollster.definitions.sample_gatherer\
.get_request_linked_samples_url(kwargs)
self.assertEqual(expected_url, url)
def test_get_request_linked_samples_url_next_sample_only_url_path(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
base_url = "http://test.com/something_that_we_do_not_care"
expected_url = "http://test.com/next_page"
kwargs = {'resource': base_url,
'next_sample_url': "/next_page"}
url = pollster.definitions.sample_gatherer\
.get_request_linked_samples_url(kwargs)
self.assertEqual(expected_url, url)

View File

@ -11,13 +11,16 @@
# License for the specific language governing permissions and limitations
# under the License.
"""Tests for ceilometer/polling/non_openstack_dynamic_pollster.py"""
"""Tests for Non-OpenStack dynamic pollsters
"""
import copy
import sys
from unittest import mock
from oslotest import base
import requests
from six.moves.urllib import parse as url_parse
from ceilometer.declarative import DynamicPollsterDefinitionException
from ceilometer.declarative import NonOpenStackApisDynamicPollsterException
@ -29,6 +32,7 @@ from ceilometer.polling.dynamic_pollster import NonOpenStackApisSamplesGatherer
from ceilometer.polling.dynamic_pollster import PollsterSampleGatherer
from ceilometer.polling.dynamic_pollster import SingleMetricPollsterDefinitions
REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit', 'value_attribute',
'url_path', 'module', 'authentication_object']
@ -116,7 +120,8 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
self.pollster_definition_only_required_fields = {
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
'value_attribute': "volume",
'url_path': "v1/test/endpoint/fake", 'module': "module-name",
'url_path': "http://server.com/v1/test/endpoint/fake",
'module': "module-name",
'authentication_object': "authentication_object"}
self.pollster_definition_all_fields = {
@ -431,3 +436,46 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
self.assertEqual(46, get_obj_sample.volume)
self.assertEqual(8, list_bucket_sample.volume)
self.assertEqual(46, put_obj_sample.volume)
def test_get_request_linked_samples_url_no_next_sample(self):
pollster = DynamicPollster(
self.pollster_definition_only_required_fields)
expected_url = self.pollster_definition_only_required_fields[
'url_path']
kwargs = {'resource': "non-openstack-resource"}
url = pollster.definitions.sample_gatherer\
.get_request_linked_samples_url(kwargs)
self.assertEqual(expected_url, url)
def test_get_request_linked_samples_url_next_sample_url(self):
pollster = DynamicPollster(
self.pollster_definition_only_required_fields)
base_url = self.pollster_definition_only_required_fields['url_path']
next_sample_path = "/next_page"
expected_url = url_parse.urljoin(base_url, next_sample_path)
kwargs = {'next_sample_url': expected_url}
url = pollster.definitions.sample_gatherer\
.get_request_linked_samples_url(kwargs)
self.assertEqual(expected_url, url)
def test_get_request_linked_samples_url_next_sample_only_url_path(self):
pollster = DynamicPollster(
self.pollster_definition_only_required_fields)
base_url = self.pollster_definition_only_required_fields['url_path']
next_sample_path = "/next_page"
expected_url = url_parse.urljoin(base_url, next_sample_path)
kwargs = {'next_sample_url': next_sample_path}
url = pollster.definitions.sample_gatherer\
.get_request_linked_samples_url(kwargs)
self.assertEqual(expected_url, url)