Merge "Dynamic pollster system to support non-OpenStack APIs"
This commit is contained in:
commit
9ed26c570a
@ -24,7 +24,7 @@ LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
class DefinitionException(Exception):
|
||||
def __init__(self, message, definition_cfg):
|
||||
def __init__(self, message, definition_cfg=None):
|
||||
msg = '%s %s: %s' % (self.__class__.__name__, definition_cfg, message)
|
||||
super(DefinitionException, self).__init__(msg)
|
||||
self.brief_message = message
|
||||
@ -46,6 +46,10 @@ class DynamicPollsterDefinitionException(DefinitionException):
|
||||
pass
|
||||
|
||||
|
||||
class NonOpenStackApisDynamicPollsterException(DefinitionException):
|
||||
pass
|
||||
|
||||
|
||||
class Definition(object):
|
||||
JSONPATH_RW_PARSER = parser.ExtentedJsonPathParser()
|
||||
GETTERS_CACHE = {}
|
||||
|
@ -0,0 +1,59 @@
|
||||
# Copyright 2014-2015 Red Hat, Inc
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from oslo_log import log
|
||||
|
||||
from ceilometer.polling.discovery.endpoint import EndpointDiscovery
|
||||
|
||||
import six.moves.urllib.parse as urlparse
|
||||
|
||||
import requests
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
class NonOpenStackCredentialsDiscovery(EndpointDiscovery):
|
||||
"""Barbican secrets discovery
|
||||
|
||||
Discovery that supplies non-OpenStack credentials for the dynamic
|
||||
pollster sub-system. This solution uses the EndpointDiscovery to
|
||||
find the Barbican URL where we can retrieve the credentials.
|
||||
"""
|
||||
|
||||
BARBICAN_URL_GET_PAYLOAD_PATTERN = "/v1/secrets/%s/payload"
|
||||
|
||||
def discover(self, manager, param=None):
|
||||
barbican_secret = "No secrets found"
|
||||
if not param:
|
||||
return [barbican_secret]
|
||||
barbican_endpoints = super(NonOpenStackCredentialsDiscovery,
|
||||
self).discover("key-manager")
|
||||
if not barbican_endpoints:
|
||||
LOG.warning("No Barbican endpoints found to execute the"
|
||||
" credentials discovery process to [%s].",
|
||||
param)
|
||||
return [barbican_secret]
|
||||
else:
|
||||
LOG.debug("Barbican endpoint found [%s].", barbican_endpoints)
|
||||
|
||||
barbican_server = next(iter(barbican_endpoints))
|
||||
barbican_endpoint = self.BARBICAN_URL_GET_PAYLOAD_PATTERN % param
|
||||
babrican_url = urlparse.urljoin(barbican_server, barbican_endpoint)
|
||||
|
||||
LOG.debug("Retrieving secrets from: %s.", babrican_url)
|
||||
resp = manager._keystone.session.get(babrican_url, authenticated=True)
|
||||
if resp.status_code != requests.codes.ok:
|
||||
resp.raise_for_status()
|
||||
|
||||
return [resp._content]
|
@ -40,20 +40,23 @@ class DynamicPollster(plugin_base.PollsterBase):
|
||||
OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values',
|
||||
'value_mapping', 'default_value',
|
||||
'metadata_mapping',
|
||||
'preserve_mapped_metadata'
|
||||
'preserve_mapped_metadata',
|
||||
'response_entries_key']
|
||||
|
||||
REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit',
|
||||
'value_attribute', 'endpoint_type',
|
||||
'url_path']
|
||||
|
||||
ALL_POLLSTER_FIELDS = OPTIONAL_POLLSTER_FIELDS + REQUIRED_POLLSTER_FIELDS
|
||||
|
||||
# Mandatory name field
|
||||
name = ""
|
||||
|
||||
def __init__(self, pollster_definitions, conf=None):
|
||||
super(DynamicPollster, self).__init__(conf)
|
||||
LOG.debug("Dynamic pollster created with [%s]",
|
||||
|
||||
self.ALL_POLLSTER_FIELDS =\
|
||||
self.OPTIONAL_POLLSTER_FIELDS + self.REQUIRED_POLLSTER_FIELDS
|
||||
|
||||
LOG.debug("%s instantiated with [%s]", __name__,
|
||||
pollster_definitions)
|
||||
|
||||
self.pollster_definitions = pollster_definitions
|
||||
@ -63,9 +66,12 @@ class DynamicPollster(plugin_base.PollsterBase):
|
||||
LOG.debug("Metadata fields configured to [%s].",
|
||||
self.pollster_definitions['metadata_fields'])
|
||||
|
||||
self.set_default_values()
|
||||
|
||||
self.name = self.pollster_definitions['name']
|
||||
self.obj = self
|
||||
|
||||
def set_default_values(self):
|
||||
if 'skip_sample_values' not in self.pollster_definitions:
|
||||
self.pollster_definitions['skip_sample_values'] = []
|
||||
|
||||
@ -113,17 +119,17 @@ class DynamicPollster(plugin_base.PollsterBase):
|
||||
LOG.debug("No resources received for processing.")
|
||||
yield None
|
||||
|
||||
for endpoint in resources:
|
||||
LOG.debug("Executing get sample on URL [%s].", endpoint)
|
||||
for r in resources:
|
||||
LOG.debug("Executing get sample for resource [%s].", r)
|
||||
|
||||
samples = list([])
|
||||
try:
|
||||
samples = self.execute_request_get_samples(
|
||||
keystone_client=manager._keystone, endpoint=endpoint)
|
||||
keystone_client=manager._keystone, resource=r)
|
||||
except RequestException as e:
|
||||
LOG.warning("Error [%s] while loading samples for [%s] "
|
||||
"for dynamic pollster [%s].",
|
||||
e, endpoint, self.name)
|
||||
e, r, self.name)
|
||||
|
||||
for pollster_sample in samples:
|
||||
response_value_attribute_name = self.pollster_definitions[
|
||||
@ -149,6 +155,10 @@ class DynamicPollster(plugin_base.PollsterBase):
|
||||
if 'project_id' in pollster_sample:
|
||||
project_id = pollster_sample["project_id"]
|
||||
|
||||
resource_id = None
|
||||
if 'id' in pollster_sample:
|
||||
resource_id = pollster_sample["id"]
|
||||
|
||||
metadata = []
|
||||
if 'metadata_fields' in self.pollster_definitions:
|
||||
metadata = dict((k, pollster_sample.get(k))
|
||||
@ -165,7 +175,7 @@ class DynamicPollster(plugin_base.PollsterBase):
|
||||
|
||||
user_id=user_id,
|
||||
project_id=project_id,
|
||||
resource_id=pollster_sample["id"],
|
||||
resource_id=resource_id,
|
||||
|
||||
resource_metadata=metadata
|
||||
)
|
||||
@ -213,12 +223,8 @@ class DynamicPollster(plugin_base.PollsterBase):
|
||||
def default_discovery(self):
|
||||
return 'endpoint:' + self.pollster_definitions['endpoint_type']
|
||||
|
||||
def execute_request_get_samples(self, keystone_client, endpoint):
|
||||
url = url_parse.urljoin(
|
||||
endpoint, self.pollster_definitions['url_path'])
|
||||
resp = keystone_client.session.get(url, authenticated=True)
|
||||
if resp.status_code != requests.codes.ok:
|
||||
resp.raise_for_status()
|
||||
def execute_request_get_samples(self, **kwargs):
|
||||
resp, url = self.internal_execute_request_get_samples(kwargs)
|
||||
|
||||
response_json = resp.json()
|
||||
|
||||
@ -231,6 +237,16 @@ class DynamicPollster(plugin_base.PollsterBase):
|
||||
return self.retrieve_entries_from_response(response_json)
|
||||
return []
|
||||
|
||||
def internal_execute_request_get_samples(self, kwargs):
|
||||
keystone_client = kwargs['keystone_client']
|
||||
endpoint = kwargs['resource']
|
||||
url = url_parse.urljoin(
|
||||
endpoint, self.pollster_definitions['url_path'])
|
||||
resp = keystone_client.session.get(url, authenticated=True)
|
||||
if resp.status_code != requests.codes.ok:
|
||||
resp.raise_for_status()
|
||||
return resp, url
|
||||
|
||||
def retrieve_entries_from_response(self, response_json):
|
||||
if isinstance(response_json, list):
|
||||
return response_json
|
||||
|
@ -40,6 +40,7 @@ from ceilometer import declarative
|
||||
from ceilometer import keystone_client
|
||||
from ceilometer import messaging
|
||||
from ceilometer.polling import dynamic_pollster
|
||||
from ceilometer.polling import non_openstack_dynamic_pollster
|
||||
from ceilometer.polling import plugin_base
|
||||
from ceilometer.publisher import utils as publisher_utils
|
||||
from ceilometer import utils
|
||||
@ -338,10 +339,8 @@ class AgentManager(cotyledon.Service):
|
||||
LOG.info("Loading dynamic pollster [%s] from file [%s].",
|
||||
pollster_name, pollsters_definitions_file)
|
||||
try:
|
||||
dynamic_pollster_object = dynamic_pollster.\
|
||||
DynamicPollster(pollster_cfg, self.conf)
|
||||
pollsters_definitions[pollster_name] = \
|
||||
dynamic_pollster_object
|
||||
pollsters_definitions[pollster_name] =\
|
||||
self.instantiate_dynamic_pollster(pollster_cfg)
|
||||
except Exception as e:
|
||||
LOG.error(
|
||||
"Error [%s] while loading dynamic pollster [%s].",
|
||||
@ -356,6 +355,13 @@ class AgentManager(cotyledon.Service):
|
||||
len(pollsters_definitions))
|
||||
return pollsters_definitions.values()
|
||||
|
||||
def instantiate_dynamic_pollster(self, pollster_cfg):
|
||||
if 'module' in pollster_cfg:
|
||||
return non_openstack_dynamic_pollster\
|
||||
.NonOpenStackApisDynamicPollster(pollster_cfg, self.conf)
|
||||
else:
|
||||
return dynamic_pollster.DynamicPollster(pollster_cfg, self.conf)
|
||||
|
||||
@staticmethod
|
||||
def _get_ext_mgr(namespace, *args, **kwargs):
|
||||
def _catch_extension_load_error(mgr, ep, exc):
|
||||
|
144
ceilometer/polling/non_openstack_dynamic_pollster.py
Normal file
144
ceilometer/polling/non_openstack_dynamic_pollster.py
Normal file
@ -0,0 +1,144 @@
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
"""Non-OpenStack Dynamic pollster component
|
||||
This component enables operators to create pollsters on the fly
|
||||
via configuration for non-OpenStack APIs. This appraoch is quite
|
||||
useful when adding metrics from APIs such as RadosGW into the Cloud
|
||||
rating and billing modules.
|
||||
"""
|
||||
import copy
|
||||
import requests
|
||||
|
||||
from ceilometer.declarative import NonOpenStackApisDynamicPollsterException
|
||||
from ceilometer.polling.dynamic_pollster import DynamicPollster
|
||||
from oslo_log import log
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
class NonOpenStackApisDynamicPollster(DynamicPollster):
|
||||
|
||||
POLLSTER_REQUIRED_POLLSTER_FIELDS = ['module', 'authentication_object']
|
||||
|
||||
POLLSTER_OPTIONAL_POLLSTER_FIELDS = ['user_id_attribute',
|
||||
'project_id_attribute',
|
||||
'resource_id_attribute',
|
||||
'barbican_secret_id',
|
||||
'authentication_parameters'
|
||||
]
|
||||
|
||||
def __init__(self, pollster_definitions, conf=None):
|
||||
# Making sure that we do not change anything in parent classes
|
||||
self.REQUIRED_POLLSTER_FIELDS = copy.deepcopy(
|
||||
DynamicPollster.REQUIRED_POLLSTER_FIELDS)
|
||||
self.OPTIONAL_POLLSTER_FIELDS = copy.deepcopy(
|
||||
DynamicPollster.OPTIONAL_POLLSTER_FIELDS)
|
||||
|
||||
# Non-OpenStack dynamic pollster do not need the 'endpoint_type'.
|
||||
self.REQUIRED_POLLSTER_FIELDS.remove('endpoint_type')
|
||||
|
||||
self.REQUIRED_POLLSTER_FIELDS += self.POLLSTER_REQUIRED_POLLSTER_FIELDS
|
||||
self.OPTIONAL_POLLSTER_FIELDS += self.POLLSTER_OPTIONAL_POLLSTER_FIELDS
|
||||
|
||||
super(NonOpenStackApisDynamicPollster, self).__init__(
|
||||
pollster_definitions, conf)
|
||||
|
||||
def set_default_values(self):
|
||||
super(NonOpenStackApisDynamicPollster, self).set_default_values()
|
||||
|
||||
if 'user_id_attribute' not in self.pollster_definitions:
|
||||
self.pollster_definitions['user_id_attribute'] = None
|
||||
|
||||
if 'project_id_attribute' not in self.pollster_definitions:
|
||||
self.pollster_definitions['project_id_attribute'] = None
|
||||
|
||||
if 'resource_id_attribute' not in self.pollster_definitions:
|
||||
self.pollster_definitions['resource_id_attribute'] = None
|
||||
|
||||
if 'barbican_secret_id' not in self.pollster_definitions:
|
||||
self.pollster_definitions['barbican_secret_id'] = ""
|
||||
|
||||
if 'authentication_parameters' not in self.pollster_definitions:
|
||||
self.pollster_definitions['authentication_parameters'] = ""
|
||||
|
||||
@property
|
||||
def default_discovery(self):
|
||||
return 'barbican:' + self.pollster_definitions['barbican_secret_id']
|
||||
|
||||
def internal_execute_request_get_samples(self, kwargs):
|
||||
credentials = kwargs['resource']
|
||||
|
||||
override_credentials = self.pollster_definitions[
|
||||
'authentication_parameters']
|
||||
if override_credentials:
|
||||
credentials = override_credentials
|
||||
|
||||
url = self.pollster_definitions['url_path']
|
||||
|
||||
authenticator_module_name = self.pollster_definitions['module']
|
||||
authenticator_class_name = \
|
||||
self.pollster_definitions['authentication_object']
|
||||
|
||||
imported_module = __import__(authenticator_module_name)
|
||||
authenticator_class = getattr(imported_module,
|
||||
authenticator_class_name)
|
||||
|
||||
authenticator_arguments = list(map(str.strip, credentials.split(",")))
|
||||
authenticator_instance = authenticator_class(*authenticator_arguments)
|
||||
|
||||
resp = requests.get(
|
||||
url,
|
||||
auth=authenticator_instance)
|
||||
|
||||
if resp.status_code != requests.codes.ok:
|
||||
raise NonOpenStackApisDynamicPollsterException(
|
||||
"Error while executing request[%s]."
|
||||
" Status[%s] and reason [%s]."
|
||||
% (url, resp.status_code, resp.reason))
|
||||
|
||||
return resp, url
|
||||
|
||||
def execute_request_get_samples(self, **kwargs):
|
||||
samples = super(NonOpenStackApisDynamicPollster,
|
||||
self).execute_request_get_samples(**kwargs)
|
||||
|
||||
if samples:
|
||||
user_id_attribute = self.pollster_definitions[
|
||||
'user_id_attribute']
|
||||
project_id_attribute = self.pollster_definitions[
|
||||
'project_id_attribute']
|
||||
resource_id_attribute = self.pollster_definitions[
|
||||
'resource_id_attribute']
|
||||
|
||||
for sample in samples:
|
||||
self.generate_new_attributes_in_sample(
|
||||
sample, user_id_attribute, 'user_id')
|
||||
self.generate_new_attributes_in_sample(
|
||||
sample, project_id_attribute, 'project_id')
|
||||
self.generate_new_attributes_in_sample(
|
||||
sample, resource_id_attribute, 'id')
|
||||
|
||||
return samples
|
||||
|
||||
def generate_new_attributes_in_sample(
|
||||
self, sample, attribute_key, new_attribute_key):
|
||||
if attribute_key:
|
||||
attribute_value = self.retrieve_attribute_nested_value(
|
||||
sample, attribute_key)
|
||||
|
||||
LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].",
|
||||
attribute_key, attribute_value, sample)
|
||||
|
||||
sample[new_attribute_key] = attribute_value
|
@ -338,9 +338,10 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
|
||||
for resource_id, samples_of_resource in resource_grouped_samples:
|
||||
# NOTE(sileht): / is forbidden by Gnocchi
|
||||
resource_id = resource_id.replace('/', '_')
|
||||
|
||||
for sample in samples_of_resource:
|
||||
metric_name = sample.name
|
||||
LOG.debug("Processing sample [%s] for resource ID [%s].",
|
||||
sample, resource_id)
|
||||
rd = self.metric_map.get(metric_name)
|
||||
if rd is None:
|
||||
if metric_name not in self._already_logged_metric_names:
|
||||
|
@ -11,7 +11,7 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Tests for ceilometer/central/manager.py
|
||||
"""Tests for ceilometer/polling/dynamic_pollster.py
|
||||
"""
|
||||
|
||||
|
||||
@ -162,7 +162,8 @@ class TestDynamicPollster(base.BaseTestCase):
|
||||
client_mock.session.get.return_value = return_value
|
||||
|
||||
samples = pollster.execute_request_get_samples(
|
||||
client_mock, "https://endpoint.server.name/")
|
||||
keystone_client=client_mock,
|
||||
resource="https://endpoint.server.name/")
|
||||
|
||||
self.assertEqual(0, len(samples))
|
||||
|
||||
@ -179,7 +180,8 @@ class TestDynamicPollster(base.BaseTestCase):
|
||||
client_mock.session.get.return_value = return_value
|
||||
|
||||
samples = pollster.execute_request_get_samples(
|
||||
client_mock, "https://endpoint.server.name/")
|
||||
keystone_client=client_mock,
|
||||
resource="https://endpoint.server.name/")
|
||||
|
||||
self.assertEqual(3, len(samples))
|
||||
|
||||
@ -196,8 +198,8 @@ class TestDynamicPollster(base.BaseTestCase):
|
||||
|
||||
exception = self.assertRaises(requests.HTTPError,
|
||||
pollster.execute_request_get_samples,
|
||||
client_mock,
|
||||
"https://endpoint.server.name/")
|
||||
keystone_client=client_mock,
|
||||
resource="https://endpoint.server.name/")
|
||||
self.assertEqual("Mock HTTP error.", str(exception))
|
||||
|
||||
def test_generate_new_metadata_fields_no_metadata_mapping(self):
|
||||
@ -325,7 +327,7 @@ class TestDynamicPollster(base.BaseTestCase):
|
||||
|
||||
self.assertEqual(0, len(samples_list))
|
||||
|
||||
def fake_sample_list(self, keystone_client=None, endpoint=None):
|
||||
def fake_sample_list(self, keystone_client=None, resource=None):
|
||||
samples_list = list()
|
||||
samples_list.append(
|
||||
{'name': "sample5", 'volume': 5, 'description': "desc-sample-5",
|
||||
|
@ -26,10 +26,14 @@ from stevedore import extension
|
||||
|
||||
from ceilometer.compute import discovery as nova_discover
|
||||
from ceilometer.hardware import discovery
|
||||
from ceilometer.polling.dynamic_pollster import DynamicPollster
|
||||
from ceilometer.polling import manager
|
||||
from ceilometer.polling.non_openstack_dynamic_pollster import \
|
||||
NonOpenStackApisDynamicPollster
|
||||
from ceilometer.polling import plugin_base
|
||||
from ceilometer import sample
|
||||
from ceilometer import service
|
||||
|
||||
from ceilometer.tests import base
|
||||
|
||||
|
||||
@ -890,3 +894,24 @@ class TestPollingAgentPartitioned(BaseAgent):
|
||||
mock.call('static_3'),
|
||||
mock.call('static_4'),
|
||||
], any_order=True)
|
||||
|
||||
def test_instantiate_dynamic_pollster_standard_pollster(self):
|
||||
pollster_definition_only_required_fields = {
|
||||
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
|
||||
'value_attribute': "volume", 'endpoint_type': "test",
|
||||
'url_path': "v1/test/endpoint/fake"}
|
||||
pollster = self.mgr.instantiate_dynamic_pollster(
|
||||
pollster_definition_only_required_fields)
|
||||
|
||||
self.assertIsInstance(pollster, DynamicPollster)
|
||||
|
||||
def test_instantiate_dynamic_pollster_non_openstack_api(self):
|
||||
pollster_definition_only_required_fields = {
|
||||
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
|
||||
'value_attribute': "volume",
|
||||
'url_path': "v1/test/endpoint/fake", 'module': "module-name",
|
||||
'authentication_object': "authentication_object"}
|
||||
pollster = self.mgr.instantiate_dynamic_pollster(
|
||||
pollster_definition_only_required_fields)
|
||||
|
||||
self.assertIsInstance(pollster, NonOpenStackApisDynamicPollster)
|
||||
|
@ -0,0 +1,115 @@
|
||||
# Copyright 2014-2015 Red Hat, Inc
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import mock
|
||||
import requests
|
||||
|
||||
from ceilometer.polling.discovery.endpoint import EndpointDiscovery
|
||||
from ceilometer.polling.discovery.non_openstack_credentials_discovery import \
|
||||
NonOpenStackCredentialsDiscovery
|
||||
from oslotest import base
|
||||
|
||||
|
||||
class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):
|
||||
|
||||
class FakeResponse(object):
|
||||
status_code = None
|
||||
json_object = None
|
||||
_content = ""
|
||||
|
||||
def json(self):
|
||||
return self.json_object
|
||||
|
||||
def raise_for_status(self):
|
||||
raise requests.HTTPError("Mock HTTP error.", response=self)
|
||||
|
||||
class FakeManager(object):
|
||||
def __init__(self, keystone_client_mock):
|
||||
self._keystone = keystone_client_mock
|
||||
|
||||
def setUp(self):
|
||||
super(TestNonOpenStackCredentialsDiscovery, self).setUp()
|
||||
|
||||
self.discovery = NonOpenStackCredentialsDiscovery(None)
|
||||
|
||||
def test_discover_no_parameters(self):
|
||||
result = self.discovery.discover(None, None)
|
||||
|
||||
self.assertEqual(['No secrets found'], result)
|
||||
|
||||
result = self.discovery.discover(None, "")
|
||||
|
||||
self.assertEqual(['No secrets found'], result)
|
||||
|
||||
def test_discover_no_barbican_endpoint(self):
|
||||
def discover_mock(self, type):
|
||||
return []
|
||||
|
||||
original_discover_method = EndpointDiscovery.discover
|
||||
EndpointDiscovery.discover = discover_mock
|
||||
|
||||
result = self.discovery.discover(None, "param")
|
||||
|
||||
self.assertEqual(['No secrets found'], result)
|
||||
|
||||
EndpointDiscovery.discover = original_discover_method
|
||||
|
||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||
def test_discover_error_response(self, client_mock):
|
||||
def discover_mock(self, type):
|
||||
return ["barbican_url"]
|
||||
|
||||
original_discover_method = EndpointDiscovery.discover
|
||||
EndpointDiscovery.discover = discover_mock
|
||||
|
||||
for http_status_code in requests.status_codes._codes.keys():
|
||||
if http_status_code < 400:
|
||||
continue
|
||||
|
||||
return_value = self.FakeResponse()
|
||||
return_value.status_code = http_status_code
|
||||
return_value.json_object = {}
|
||||
|
||||
client_mock.session.get.return_value = return_value
|
||||
|
||||
exception = self.assertRaises(
|
||||
requests.HTTPError,
|
||||
self.discovery.discover,
|
||||
manager=self.FakeManager(client_mock),
|
||||
param="param")
|
||||
|
||||
self.assertEqual("Mock HTTP error.", str(exception))
|
||||
|
||||
EndpointDiscovery.discover = original_discover_method
|
||||
|
||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||
def test_discover_response_ok(self, client_mock):
|
||||
def discover_mock(self, type):
|
||||
return ["barbican_url"]
|
||||
|
||||
original_discover_method = EndpointDiscovery.discover
|
||||
EndpointDiscovery.discover = discover_mock
|
||||
|
||||
return_value = self.FakeResponse()
|
||||
return_value.status_code = requests.codes.ok
|
||||
return_value.json_object = {}
|
||||
return_value._content = "content"
|
||||
|
||||
client_mock.session.get.return_value = return_value
|
||||
|
||||
response = self.discovery.discover(
|
||||
manager=self.FakeManager(client_mock), param="param")
|
||||
|
||||
self.assertEqual(["content"], response)
|
||||
|
||||
EndpointDiscovery.discover = original_discover_method
|
@ -0,0 +1,273 @@
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Tests for ceilometer/polling/non_openstack_dynamic_pollster.py
|
||||
"""
|
||||
import copy
|
||||
import sys
|
||||
|
||||
import mock
|
||||
import requests
|
||||
|
||||
from ceilometer.declarative import DynamicPollsterDefinitionException
|
||||
from ceilometer.declarative import NonOpenStackApisDynamicPollsterException
|
||||
from ceilometer.polling.dynamic_pollster import DynamicPollster
|
||||
from ceilometer.polling.non_openstack_dynamic_pollster\
|
||||
import NonOpenStackApisDynamicPollster
|
||||
|
||||
from oslotest import base
|
||||
|
||||
|
||||
class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
|
||||
|
||||
class FakeResponse(object):
|
||||
status_code = None
|
||||
json_object = None
|
||||
|
||||
def json(self):
|
||||
return self.json_object
|
||||
|
||||
def raise_for_status(self):
|
||||
raise requests.HTTPError("Mock HTTP error.", response=self)
|
||||
|
||||
def setUp(self):
|
||||
super(TestNonOpenStackApisDynamicPollster, self).setUp()
|
||||
self.pollster_definition_only_required_fields = {
|
||||
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
|
||||
'value_attribute': "volume",
|
||||
'url_path': "v1/test/endpoint/fake", 'module': "module-name",
|
||||
'authentication_object': "authentication_object"}
|
||||
|
||||
self.pollster_definition_all_fields = {
|
||||
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
|
||||
'value_attribute': "volume",
|
||||
'url_path': "v1/test/endpoint/fake", 'module': "module-name",
|
||||
'authentication_object': "authentication_object",
|
||||
'user_id_attribute': 'user_id',
|
||||
'project_id_attribute': 'project_id',
|
||||
'resource_id_attribute': 'id', 'barbican_secret_id': 'barbican_id',
|
||||
'authentication_parameters': 'parameters'}
|
||||
|
||||
def test_all_fields(self):
|
||||
pollster = NonOpenStackApisDynamicPollster(
|
||||
self.pollster_definition_only_required_fields)
|
||||
|
||||
all_required = ['module', 'authentication_object', 'name',
|
||||
'sample_type', 'unit', 'value_attribute',
|
||||
'url_path']
|
||||
|
||||
all_optional = ['metadata_fields', 'skip_sample_values',
|
||||
'value_mapping', 'default_value', 'metadata_mapping',
|
||||
'preserve_mapped_metadata', 'user_id_attribute',
|
||||
'project_id_attribute', 'resource_id_attribute',
|
||||
'barbican_secret_id', 'authentication_parameters',
|
||||
'response_entries_key'] + all_required
|
||||
|
||||
for field in all_required:
|
||||
self.assertIn(field, pollster.REQUIRED_POLLSTER_FIELDS)
|
||||
|
||||
for field in all_optional:
|
||||
self.assertIn(field, pollster.ALL_POLLSTER_FIELDS)
|
||||
|
||||
def test_all_required_fields_exceptions(self):
|
||||
pollster = NonOpenStackApisDynamicPollster(
|
||||
self.pollster_definition_only_required_fields)
|
||||
|
||||
for key in pollster.REQUIRED_POLLSTER_FIELDS:
|
||||
pollster_definition = copy.deepcopy(
|
||||
self.pollster_definition_only_required_fields)
|
||||
pollster_definition.pop(key)
|
||||
exception = self.assertRaises(DynamicPollsterDefinitionException,
|
||||
NonOpenStackApisDynamicPollster,
|
||||
pollster_definition)
|
||||
self.assertEqual("Required fields ['%s'] not specified."
|
||||
% key, exception.brief_message)
|
||||
|
||||
def test_set_default_values(self):
|
||||
pollster = NonOpenStackApisDynamicPollster(
|
||||
self.pollster_definition_only_required_fields)
|
||||
|
||||
pollster_definitions = pollster.pollster_definitions
|
||||
|
||||
self.assertEqual(None, pollster_definitions['user_id_attribute'])
|
||||
self.assertEqual(None, pollster_definitions['project_id_attribute'])
|
||||
self.assertEqual(None, pollster_definitions['resource_id_attribute'])
|
||||
self.assertEqual('', pollster_definitions['barbican_secret_id'])
|
||||
self.assertEqual('', pollster_definitions['authentication_parameters'])
|
||||
|
||||
def test_user_set_optional_parameters(self):
|
||||
pollster = NonOpenStackApisDynamicPollster(
|
||||
self.pollster_definition_all_fields)
|
||||
pollster_definitions = pollster.pollster_definitions
|
||||
|
||||
self.assertEqual('user_id',
|
||||
pollster_definitions['user_id_attribute'])
|
||||
self.assertEqual('project_id',
|
||||
pollster_definitions['project_id_attribute'])
|
||||
self.assertEqual('id',
|
||||
pollster_definitions['resource_id_attribute'])
|
||||
self.assertEqual('barbican_id',
|
||||
pollster_definitions['barbican_secret_id'])
|
||||
self.assertEqual('parameters',
|
||||
pollster_definitions['authentication_parameters'])
|
||||
|
||||
def test_default_discovery_empty_secret_id(self):
|
||||
pollster = NonOpenStackApisDynamicPollster(
|
||||
self.pollster_definition_only_required_fields)
|
||||
|
||||
self.assertEqual("barbican:", pollster.default_discovery)
|
||||
|
||||
def test_default_discovery_not_empty_secret_id(self):
|
||||
pollster = NonOpenStackApisDynamicPollster(
|
||||
self.pollster_definition_all_fields)
|
||||
|
||||
self.assertEqual("barbican:barbican_id", pollster.default_discovery)
|
||||
|
||||
@mock.patch('requests.get')
|
||||
def test_internal_execute_request_get_samples_status_code_ok(
|
||||
self, get_mock):
|
||||
sys.modules['module-name'] = mock.MagicMock()
|
||||
|
||||
pollster = NonOpenStackApisDynamicPollster(
|
||||
self.pollster_definition_only_required_fields)
|
||||
|
||||
return_value = self.FakeResponse()
|
||||
return_value.status_code = requests.codes.ok
|
||||
return_value.json_object = {}
|
||||
return_value.reason = "Ok"
|
||||
|
||||
get_mock.return_value = return_value
|
||||
|
||||
kwargs = {'resource': "credentials"}
|
||||
|
||||
resp, url = pollster.internal_execute_request_get_samples(kwargs)
|
||||
|
||||
self.assertEqual(
|
||||
self.pollster_definition_only_required_fields['url_path'], url)
|
||||
self.assertEqual(return_value, resp)
|
||||
|
||||
@mock.patch('requests.get')
|
||||
def test_internal_execute_request_get_samples_status_code_not_ok(
|
||||
self, get_mock):
|
||||
sys.modules['module-name'] = mock.MagicMock()
|
||||
|
||||
pollster = NonOpenStackApisDynamicPollster(
|
||||
self.pollster_definition_only_required_fields)
|
||||
|
||||
for http_status_code in requests.status_codes._codes.keys():
|
||||
if http_status_code >= 400:
|
||||
return_value = self.FakeResponse()
|
||||
return_value.status_code = http_status_code
|
||||
return_value.json_object = {}
|
||||
return_value.reason = requests.status_codes._codes[
|
||||
http_status_code][0]
|
||||
|
||||
get_mock.return_value = return_value
|
||||
|
||||
kwargs = {'resource': "credentials"}
|
||||
exception = self.assertRaises(
|
||||
NonOpenStackApisDynamicPollsterException,
|
||||
pollster.internal_execute_request_get_samples, kwargs)
|
||||
|
||||
self.assertEqual(
|
||||
"NonOpenStackApisDynamicPollsterException"
|
||||
" None: Error while executing request[%s]."
|
||||
" Status[%s] and reason [%s]."
|
||||
%
|
||||
(self.pollster_definition_only_required_fields['url_path'],
|
||||
http_status_code, return_value.reason), str(exception))
|
||||
|
||||
def test_generate_new_attributes_in_sample_attribute_key_none(self):
|
||||
pollster = NonOpenStackApisDynamicPollster(
|
||||
self.pollster_definition_only_required_fields)
|
||||
|
||||
sample = {"test": "2"}
|
||||
new_key = "new-key"
|
||||
|
||||
pollster.generate_new_attributes_in_sample(sample, None, new_key)
|
||||
pollster.generate_new_attributes_in_sample(sample, "", new_key)
|
||||
|
||||
self.assertNotIn(new_key, sample)
|
||||
|
||||
def test_generate_new_attributes_in_sample(self):
|
||||
pollster = NonOpenStackApisDynamicPollster(
|
||||
self.pollster_definition_only_required_fields)
|
||||
|
||||
sample = {"test": "2"}
|
||||
new_key = "new-key"
|
||||
|
||||
pollster.generate_new_attributes_in_sample(sample, "test", new_key)
|
||||
|
||||
self.assertIn(new_key, sample)
|
||||
self.assertEqual(sample["test"], sample[new_key])
|
||||
|
||||
def test_execute_request_get_samples_non_empty_keys(self):
|
||||
sample = {'user_id_attribute': "123456789",
|
||||
'project_id_attribute': "dfghyt432345t",
|
||||
'resource_id_attribute': "sdfghjt543"}
|
||||
|
||||
def execute_request_get_samples_mock(self, **kwargs):
|
||||
samples = [sample]
|
||||
return samples
|
||||
|
||||
DynamicPollster.execute_request_get_samples =\
|
||||
execute_request_get_samples_mock
|
||||
|
||||
self.pollster_definition_all_fields[
|
||||
'user_id_attribute'] = 'user_id_attribute'
|
||||
self.pollster_definition_all_fields[
|
||||
'project_id_attribute'] = 'project_id_attribute'
|
||||
self.pollster_definition_all_fields[
|
||||
'resource_id_attribute'] = 'resource_id_attribute'
|
||||
|
||||
pollster = NonOpenStackApisDynamicPollster(
|
||||
self.pollster_definition_all_fields)
|
||||
|
||||
params = {"d": "d"}
|
||||
response = pollster.execute_request_get_samples(**params)
|
||||
|
||||
self.assertEqual(sample['user_id_attribute'],
|
||||
response[0]['user_id'])
|
||||
self.assertEqual(sample['project_id_attribute'],
|
||||
response[0]['project_id'])
|
||||
self.assertEqual(sample['resource_id_attribute'],
|
||||
response[0]['id'])
|
||||
|
||||
def test_execute_request_get_samples_empty_keys(self):
|
||||
sample = {'user_id_attribute': "123456789",
|
||||
'project_id_attribute': "dfghyt432345t",
|
||||
'resource_id_attribute': "sdfghjt543"}
|
||||
|
||||
def execute_request_get_samples_mock(self, **kwargs):
|
||||
samples = [sample]
|
||||
return samples
|
||||
|
||||
DynamicPollster.execute_request_get_samples =\
|
||||
execute_request_get_samples_mock
|
||||
|
||||
self.pollster_definition_all_fields[
|
||||
'user_id_attribute'] = None
|
||||
self.pollster_definition_all_fields[
|
||||
'project_id_attribute'] = None
|
||||
self.pollster_definition_all_fields[
|
||||
'resource_id_attribute'] = None
|
||||
|
||||
pollster = NonOpenStackApisDynamicPollster(
|
||||
self.pollster_definition_all_fields)
|
||||
|
||||
params = {"d": "d"}
|
||||
response = pollster.execute_request_get_samples(**params)
|
||||
|
||||
self.assertNotIn('user_id', response[0])
|
||||
self.assertNotIn('project_id', response[0])
|
||||
self.assertNotIn('id', response[0])
|
@ -618,6 +618,8 @@ class PublisherWorkflowTest(base.BaseTestCase,
|
||||
expected_debug = [
|
||||
mock.call('filtered project found: %s',
|
||||
'a2d42c23-d518-46b6-96ab-3fba2e146859'),
|
||||
mock.call('Processing sample [%s] for resource ID [%s].',
|
||||
self.sample, resource_id),
|
||||
]
|
||||
|
||||
measures_posted = False
|
||||
|
@ -24,14 +24,12 @@ dynamic pollster system:
|
||||
fashion. This feature is "a nice" to have, but is currently not
|
||||
implemented.
|
||||
|
||||
* non-OpenStack APIs such as RadosGW (currently in development)
|
||||
|
||||
* APIs that return a list of entries directly, without a first key for the
|
||||
list. An example is Aodh alarm list.
|
||||
|
||||
|
||||
The dynamic pollsters system configuration
|
||||
------------------------------------------
|
||||
The dynamic pollsters system configuration (for OpenStack APIs)
|
||||
---------------------------------------------------------------
|
||||
Each YAML file in the dynamic pollster feature can use the following
|
||||
attributes to define a dynamic pollster:
|
||||
|
||||
@ -243,3 +241,91 @@ desires):
|
||||
metadata_mapping:
|
||||
name: "display_name"
|
||||
default_value: 0
|
||||
|
||||
The dynamic pollsters system configuration (for non-OpenStack APIs)
|
||||
-------------------------------------------------------------------
|
||||
|
||||
The dynamic pollster system can also be used for non-OpenStack APIs.
|
||||
to configure non-OpenStack APIs, one can use all but one attribute of
|
||||
the Dynamic pollster system. The attribute that is not supported is
|
||||
the ``endpoint_type``. The dynamic pollster system for non-OpenStack APIs
|
||||
is activated automatically when one uses the configurations ``module``.
|
||||
|
||||
The extra parameters that are available when using the Non-OpenStack
|
||||
dynamic pollster sub-subsystem are the following:
|
||||
|
||||
* ``module``: required parameter. It is the python module name that Ceilometer
|
||||
has to load to use the authentication object when executing requests against
|
||||
the API. For instance, if one wants to create a pollster to gather data from
|
||||
RadosGW, he/she can use the ``awsauth`` python module.
|
||||
|
||||
* ``authentication_object``: mandatory parameter. The name of the class that we
|
||||
can find in the ``module`` that Ceilometer will use as the authentication
|
||||
object in the request. For instance, when using the ``awsauth`` python module
|
||||
to gather data from RadosGW, one can use the authentication object as
|
||||
``S3Auth``.
|
||||
|
||||
* ``authentication_parameters``: optional parameter. It is a comma separated
|
||||
value that will be used to instantiate the ``authentication_object``. For
|
||||
instance, if we gather data from RadosGW, and we use the ``S3Auth`` class,
|
||||
the ``authentication_parameters`` can be configured as
|
||||
``<rados_gw_access_key>, rados_gw_secret_key, rados_gw_host_name``.
|
||||
|
||||
* ``barbican_secret_id``: optional parameter. The Barbican secret ID,
|
||||
from which, Ceilometer can retrieve the comma separated values of the
|
||||
``authentication_parameters``.
|
||||
|
||||
* ``user_id_attribute``: optional parameter. The name of the attribute in the
|
||||
entries that are processed from ``response_entries_key`` elements that
|
||||
will be mapped to ``user_id`` attribute that is sent to Gnocchi.
|
||||
|
||||
* ``project_id_attribute``: optional parameter. The name of the attribute in
|
||||
the entries that are processed from ``response_entries_key`` elements that
|
||||
will be mapped to ``project_id`` attribute that is sent to Gnocchi.
|
||||
|
||||
* ``resource_id_attribute``: optional parameter. The name of the attribute
|
||||
in the entries that are processed from ``response_entries_key`` elements that
|
||||
will be mapped to ``id`` attribute that is sent to Gnocchi.
|
||||
|
||||
As follows we present an example on how to convert the hard-coded pollster
|
||||
for `radosgw.api.request` metric to the dynamic pollster model:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
---
|
||||
|
||||
- name: "dynamic.radosgw.api.request"
|
||||
sample_type: "gauge"
|
||||
unit: "request"
|
||||
value_attribute: "total.ops"
|
||||
url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage"
|
||||
module: "awsauth"
|
||||
authentication_object: "S3Auth"
|
||||
authentication_parameters: "<access_key>, <secret_key>,
|
||||
<rados_gateway_server>"
|
||||
user_id_attribute: "user"
|
||||
project_id_attribute: "user"
|
||||
resource_id_attribute: "user"
|
||||
response_entries_key: "summary"
|
||||
|
||||
We can take that example a bit further, and instead of gathering the `total
|
||||
.ops` variable, which counts for all the requests (even the unsuccessful
|
||||
ones), we can use the `successful_ops`.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
---
|
||||
|
||||
- name: "dynamic.radosgw.api.request.successful_ops"
|
||||
sample_type: "gauge"
|
||||
unit: "request"
|
||||
value_attribute: "total.successful_ops"
|
||||
url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage"
|
||||
module: "awsauth"
|
||||
authentication_object: "S3Auth"
|
||||
authentication_parameters: "<access_key>, <secret_key>,
|
||||
<rados_gateway_server>"
|
||||
user_id_attribute: "user"
|
||||
project_id_attribute: "user"
|
||||
resource_id_attribute: "user"
|
||||
response_entries_key: "summary"
|
||||
|
@ -0,0 +1,6 @@
|
||||
---
|
||||
features:
|
||||
- |
|
||||
Add the support for non-OpenStack APIs in the dynamic pollster system.
|
||||
This extension enables operators to create pollster on the fly to handle
|
||||
metrics from systems such as the RadosGW API.
|
@ -50,6 +50,7 @@ ceilometer.discover.compute =
|
||||
local_instances = ceilometer.compute.discovery:InstanceDiscovery
|
||||
|
||||
ceilometer.discover.central =
|
||||
barbican = ceilometer.polling.discovery.non_openstack_credentials_discovery:NonOpenStackCredentialsDiscovery
|
||||
endpoint = ceilometer.polling.discovery.endpoint:EndpointDiscovery
|
||||
tenant = ceilometer.polling.discovery.tenant:TenantDiscovery
|
||||
lb_pools = ceilometer.network.services.discovery:LBPoolsDiscovery
|
||||
|
Loading…
Reference in New Issue
Block a user