Browse Source

Merge "Dynamic pollster system to support non-OpenStack APIs"

changes/19/694519/7
Zuul 2 weeks ago
parent
commit
9ed26c570a
14 changed files with 771 additions and 31 deletions
  1. +5
    -1
      ceilometer/declarative.py
  2. +59
    -0
      ceilometer/polling/discovery/non_openstack_credentials_discovery.py
  3. +31
    -15
      ceilometer/polling/dynamic_pollster.py
  4. +10
    -4
      ceilometer/polling/manager.py
  5. +144
    -0
      ceilometer/polling/non_openstack_dynamic_pollster.py
  6. +2
    -1
      ceilometer/publisher/gnocchi.py
  7. +8
    -6
      ceilometer/tests/unit/polling/test_dynamic_pollster.py
  8. +25
    -0
      ceilometer/tests/unit/polling/test_manager.py
  9. +115
    -0
      ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py
  10. +273
    -0
      ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
  11. +2
    -0
      ceilometer/tests/unit/publisher/test_gnocchi.py
  12. +90
    -4
      doc/source/admin/telemetry-dynamic-pollster.rst
  13. +6
    -0
      releasenotes/notes/dynamic-pollster-system-for-non-openstack-apis-4e06694f223f34f3.yaml
  14. +1
    -0
      setup.cfg

+ 5
- 1
ceilometer/declarative.py View File

@@ -24,7 +24,7 @@ LOG = log.getLogger(__name__)


class DefinitionException(Exception):
def __init__(self, message, definition_cfg):
def __init__(self, message, definition_cfg=None):
msg = '%s %s: %s' % (self.__class__.__name__, definition_cfg, message)
super(DefinitionException, self).__init__(msg)
self.brief_message = message
@@ -46,6 +46,10 @@ class DynamicPollsterDefinitionException(DefinitionException):
pass


class NonOpenStackApisDynamicPollsterException(DefinitionException):
pass


class Definition(object):
JSONPATH_RW_PARSER = parser.ExtentedJsonPathParser()
GETTERS_CACHE = {}

+ 59
- 0
ceilometer/polling/discovery/non_openstack_credentials_discovery.py View File

@@ -0,0 +1,59 @@
# Copyright 2014-2015 Red Hat, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from oslo_log import log

from ceilometer.polling.discovery.endpoint import EndpointDiscovery

import six.moves.urllib.parse as urlparse

import requests

LOG = log.getLogger(__name__)


class NonOpenStackCredentialsDiscovery(EndpointDiscovery):
"""Barbican secrets discovery

Discovery that supplies non-OpenStack credentials for the dynamic
pollster sub-system. This solution uses the EndpointDiscovery to
find the Barbican URL where we can retrieve the credentials.
"""

BARBICAN_URL_GET_PAYLOAD_PATTERN = "/v1/secrets/%s/payload"

def discover(self, manager, param=None):
barbican_secret = "No secrets found"
if not param:
return [barbican_secret]
barbican_endpoints = super(NonOpenStackCredentialsDiscovery,
self).discover("key-manager")
if not barbican_endpoints:
LOG.warning("No Barbican endpoints found to execute the"
" credentials discovery process to [%s].",
param)
return [barbican_secret]
else:
LOG.debug("Barbican endpoint found [%s].", barbican_endpoints)

barbican_server = next(iter(barbican_endpoints))
barbican_endpoint = self.BARBICAN_URL_GET_PAYLOAD_PATTERN % param
babrican_url = urlparse.urljoin(barbican_server, barbican_endpoint)

LOG.debug("Retrieving secrets from: %s.", babrican_url)
resp = manager._keystone.session.get(babrican_url, authenticated=True)
if resp.status_code != requests.codes.ok:
resp.raise_for_status()

return [resp._content]

+ 31
- 15
ceilometer/polling/dynamic_pollster.py View File

@@ -40,20 +40,23 @@ class DynamicPollster(plugin_base.PollsterBase):
OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values',
'value_mapping', 'default_value',
'metadata_mapping',
'preserve_mapped_metadata'
'preserve_mapped_metadata',
'response_entries_key']

REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit',
'value_attribute', 'endpoint_type',
'url_path']

ALL_POLLSTER_FIELDS = OPTIONAL_POLLSTER_FIELDS + REQUIRED_POLLSTER_FIELDS

# Mandatory name field
name = ""

def __init__(self, pollster_definitions, conf=None):
super(DynamicPollster, self).__init__(conf)
LOG.debug("Dynamic pollster created with [%s]",

self.ALL_POLLSTER_FIELDS =\
self.OPTIONAL_POLLSTER_FIELDS + self.REQUIRED_POLLSTER_FIELDS

LOG.debug("%s instantiated with [%s]", __name__,
pollster_definitions)

self.pollster_definitions = pollster_definitions
@@ -63,9 +66,12 @@ class DynamicPollster(plugin_base.PollsterBase):
LOG.debug("Metadata fields configured to [%s].",
self.pollster_definitions['metadata_fields'])

self.set_default_values()

self.name = self.pollster_definitions['name']
self.obj = self

def set_default_values(self):
if 'skip_sample_values' not in self.pollster_definitions:
self.pollster_definitions['skip_sample_values'] = []

@@ -113,17 +119,17 @@ class DynamicPollster(plugin_base.PollsterBase):
LOG.debug("No resources received for processing.")
yield None

for endpoint in resources:
LOG.debug("Executing get sample on URL [%s].", endpoint)
for r in resources:
LOG.debug("Executing get sample for resource [%s].", r)

samples = list([])
try:
samples = self.execute_request_get_samples(
keystone_client=manager._keystone, endpoint=endpoint)
keystone_client=manager._keystone, resource=r)
except RequestException as e:
LOG.warning("Error [%s] while loading samples for [%s] "
"for dynamic pollster [%s].",
e, endpoint, self.name)
e, r, self.name)

for pollster_sample in samples:
response_value_attribute_name = self.pollster_definitions[
@@ -149,6 +155,10 @@ class DynamicPollster(plugin_base.PollsterBase):
if 'project_id' in pollster_sample:
project_id = pollster_sample["project_id"]

resource_id = None
if 'id' in pollster_sample:
resource_id = pollster_sample["id"]

metadata = []
if 'metadata_fields' in self.pollster_definitions:
metadata = dict((k, pollster_sample.get(k))
@@ -165,7 +175,7 @@ class DynamicPollster(plugin_base.PollsterBase):

user_id=user_id,
project_id=project_id,
resource_id=pollster_sample["id"],
resource_id=resource_id,

resource_metadata=metadata
)
@@ -213,12 +223,8 @@ class DynamicPollster(plugin_base.PollsterBase):
def default_discovery(self):
return 'endpoint:' + self.pollster_definitions['endpoint_type']

def execute_request_get_samples(self, keystone_client, endpoint):
url = url_parse.urljoin(
endpoint, self.pollster_definitions['url_path'])
resp = keystone_client.session.get(url, authenticated=True)
if resp.status_code != requests.codes.ok:
resp.raise_for_status()
def execute_request_get_samples(self, **kwargs):
resp, url = self.internal_execute_request_get_samples(kwargs)

response_json = resp.json()

@@ -231,6 +237,16 @@ class DynamicPollster(plugin_base.PollsterBase):
return self.retrieve_entries_from_response(response_json)
return []

def internal_execute_request_get_samples(self, kwargs):
keystone_client = kwargs['keystone_client']
endpoint = kwargs['resource']
url = url_parse.urljoin(
endpoint, self.pollster_definitions['url_path'])
resp = keystone_client.session.get(url, authenticated=True)
if resp.status_code != requests.codes.ok:
resp.raise_for_status()
return resp, url

def retrieve_entries_from_response(self, response_json):
if isinstance(response_json, list):
return response_json

+ 10
- 4
ceilometer/polling/manager.py View File

@@ -40,6 +40,7 @@ from ceilometer import declarative
from ceilometer import keystone_client
from ceilometer import messaging
from ceilometer.polling import dynamic_pollster
from ceilometer.polling import non_openstack_dynamic_pollster
from ceilometer.polling import plugin_base
from ceilometer.publisher import utils as publisher_utils
from ceilometer import utils
@@ -338,10 +339,8 @@ class AgentManager(cotyledon.Service):
LOG.info("Loading dynamic pollster [%s] from file [%s].",
pollster_name, pollsters_definitions_file)
try:
dynamic_pollster_object = dynamic_pollster.\
DynamicPollster(pollster_cfg, self.conf)
pollsters_definitions[pollster_name] = \
dynamic_pollster_object
pollsters_definitions[pollster_name] =\
self.instantiate_dynamic_pollster(pollster_cfg)
except Exception as e:
LOG.error(
"Error [%s] while loading dynamic pollster [%s].",
@@ -356,6 +355,13 @@ class AgentManager(cotyledon.Service):
len(pollsters_definitions))
return pollsters_definitions.values()

def instantiate_dynamic_pollster(self, pollster_cfg):
if 'module' in pollster_cfg:
return non_openstack_dynamic_pollster\
.NonOpenStackApisDynamicPollster(pollster_cfg, self.conf)
else:
return dynamic_pollster.DynamicPollster(pollster_cfg, self.conf)

@staticmethod
def _get_ext_mgr(namespace, *args, **kwargs):
def _catch_extension_load_error(mgr, ep, exc):

+ 144
- 0
ceilometer/polling/non_openstack_dynamic_pollster.py View File

@@ -0,0 +1,144 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.


"""Non-OpenStack Dynamic pollster component
This component enables operators to create pollsters on the fly
via configuration for non-OpenStack APIs. This appraoch is quite
useful when adding metrics from APIs such as RadosGW into the Cloud
rating and billing modules.
"""
import copy
import requests

from ceilometer.declarative import NonOpenStackApisDynamicPollsterException
from ceilometer.polling.dynamic_pollster import DynamicPollster
from oslo_log import log

LOG = log.getLogger(__name__)


class NonOpenStackApisDynamicPollster(DynamicPollster):

POLLSTER_REQUIRED_POLLSTER_FIELDS = ['module', 'authentication_object']

POLLSTER_OPTIONAL_POLLSTER_FIELDS = ['user_id_attribute',
'project_id_attribute',
'resource_id_attribute',
'barbican_secret_id',
'authentication_parameters'
]

def __init__(self, pollster_definitions, conf=None):
# Making sure that we do not change anything in parent classes
self.REQUIRED_POLLSTER_FIELDS = copy.deepcopy(
DynamicPollster.REQUIRED_POLLSTER_FIELDS)
self.OPTIONAL_POLLSTER_FIELDS = copy.deepcopy(
DynamicPollster.OPTIONAL_POLLSTER_FIELDS)

# Non-OpenStack dynamic pollster do not need the 'endpoint_type'.
self.REQUIRED_POLLSTER_FIELDS.remove('endpoint_type')

self.REQUIRED_POLLSTER_FIELDS += self.POLLSTER_REQUIRED_POLLSTER_FIELDS
self.OPTIONAL_POLLSTER_FIELDS += self.POLLSTER_OPTIONAL_POLLSTER_FIELDS

super(NonOpenStackApisDynamicPollster, self).__init__(
pollster_definitions, conf)

def set_default_values(self):
super(NonOpenStackApisDynamicPollster, self).set_default_values()

if 'user_id_attribute' not in self.pollster_definitions:
self.pollster_definitions['user_id_attribute'] = None

if 'project_id_attribute' not in self.pollster_definitions:
self.pollster_definitions['project_id_attribute'] = None

if 'resource_id_attribute' not in self.pollster_definitions:
self.pollster_definitions['resource_id_attribute'] = None

if 'barbican_secret_id' not in self.pollster_definitions:
self.pollster_definitions['barbican_secret_id'] = ""

if 'authentication_parameters' not in self.pollster_definitions:
self.pollster_definitions['authentication_parameters'] = ""

@property
def default_discovery(self):
return 'barbican:' + self.pollster_definitions['barbican_secret_id']

def internal_execute_request_get_samples(self, kwargs):
credentials = kwargs['resource']

override_credentials = self.pollster_definitions[
'authentication_parameters']
if override_credentials:
credentials = override_credentials

url = self.pollster_definitions['url_path']

authenticator_module_name = self.pollster_definitions['module']
authenticator_class_name = \
self.pollster_definitions['authentication_object']

imported_module = __import__(authenticator_module_name)
authenticator_class = getattr(imported_module,
authenticator_class_name)

authenticator_arguments = list(map(str.strip, credentials.split(",")))
authenticator_instance = authenticator_class(*authenticator_arguments)

resp = requests.get(
url,
auth=authenticator_instance)

if resp.status_code != requests.codes.ok:
raise NonOpenStackApisDynamicPollsterException(
"Error while executing request[%s]."
" Status[%s] and reason [%s]."
% (url, resp.status_code, resp.reason))

return resp, url

def execute_request_get_samples(self, **kwargs):
samples = super(NonOpenStackApisDynamicPollster,
self).execute_request_get_samples(**kwargs)

if samples:
user_id_attribute = self.pollster_definitions[
'user_id_attribute']
project_id_attribute = self.pollster_definitions[
'project_id_attribute']
resource_id_attribute = self.pollster_definitions[
'resource_id_attribute']

for sample in samples:
self.generate_new_attributes_in_sample(
sample, user_id_attribute, 'user_id')
self.generate_new_attributes_in_sample(
sample, project_id_attribute, 'project_id')
self.generate_new_attributes_in_sample(
sample, resource_id_attribute, 'id')

return samples

def generate_new_attributes_in_sample(
self, sample, attribute_key, new_attribute_key):
if attribute_key:
attribute_value = self.retrieve_attribute_nested_value(
sample, attribute_key)

LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].",
attribute_key, attribute_value, sample)

sample[new_attribute_key] = attribute_value

+ 2
- 1
ceilometer/publisher/gnocchi.py View File

@@ -338,9 +338,10 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
for resource_id, samples_of_resource in resource_grouped_samples:
# NOTE(sileht): / is forbidden by Gnocchi
resource_id = resource_id.replace('/', '_')

for sample in samples_of_resource:
metric_name = sample.name
LOG.debug("Processing sample [%s] for resource ID [%s].",
sample, resource_id)
rd = self.metric_map.get(metric_name)
if rd is None:
if metric_name not in self._already_logged_metric_names:

+ 8
- 6
ceilometer/tests/unit/polling/test_dynamic_pollster.py View File

@@ -11,7 +11,7 @@
# License for the specific language governing permissions and limitations
# under the License.

"""Tests for ceilometer/central/manager.py
"""Tests for ceilometer/polling/dynamic_pollster.py
"""


@@ -162,7 +162,8 @@ class TestDynamicPollster(base.BaseTestCase):
client_mock.session.get.return_value = return_value

samples = pollster.execute_request_get_samples(
client_mock, "https://endpoint.server.name/")
keystone_client=client_mock,
resource="https://endpoint.server.name/")

self.assertEqual(0, len(samples))

@@ -179,7 +180,8 @@ class TestDynamicPollster(base.BaseTestCase):
client_mock.session.get.return_value = return_value

samples = pollster.execute_request_get_samples(
client_mock, "https://endpoint.server.name/")
keystone_client=client_mock,
resource="https://endpoint.server.name/")

self.assertEqual(3, len(samples))

@@ -196,8 +198,8 @@ class TestDynamicPollster(base.BaseTestCase):

exception = self.assertRaises(requests.HTTPError,
pollster.execute_request_get_samples,
client_mock,
"https://endpoint.server.name/")
keystone_client=client_mock,
resource="https://endpoint.server.name/")
self.assertEqual("Mock HTTP error.", str(exception))

def test_generate_new_metadata_fields_no_metadata_mapping(self):
@@ -325,7 +327,7 @@ class TestDynamicPollster(base.BaseTestCase):

self.assertEqual(0, len(samples_list))

def fake_sample_list(self, keystone_client=None, endpoint=None):
def fake_sample_list(self, keystone_client=None, resource=None):
samples_list = list()
samples_list.append(
{'name': "sample5", 'volume': 5, 'description': "desc-sample-5",

+ 25
- 0
ceilometer/tests/unit/polling/test_manager.py View File

@@ -26,10 +26,14 @@ from stevedore import extension

from ceilometer.compute import discovery as nova_discover
from ceilometer.hardware import discovery
from ceilometer.polling.dynamic_pollster import DynamicPollster
from ceilometer.polling import manager
from ceilometer.polling.non_openstack_dynamic_pollster import \
NonOpenStackApisDynamicPollster
from ceilometer.polling import plugin_base
from ceilometer import sample
from ceilometer import service

from ceilometer.tests import base


@@ -890,3 +894,24 @@ class TestPollingAgentPartitioned(BaseAgent):
mock.call('static_3'),
mock.call('static_4'),
], any_order=True)

def test_instantiate_dynamic_pollster_standard_pollster(self):
pollster_definition_only_required_fields = {
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
'value_attribute': "volume", 'endpoint_type': "test",
'url_path': "v1/test/endpoint/fake"}
pollster = self.mgr.instantiate_dynamic_pollster(
pollster_definition_only_required_fields)

self.assertIsInstance(pollster, DynamicPollster)

def test_instantiate_dynamic_pollster_non_openstack_api(self):
pollster_definition_only_required_fields = {
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
'value_attribute': "volume",
'url_path': "v1/test/endpoint/fake", 'module': "module-name",
'authentication_object': "authentication_object"}
pollster = self.mgr.instantiate_dynamic_pollster(
pollster_definition_only_required_fields)

self.assertIsInstance(pollster, NonOpenStackApisDynamicPollster)

+ 115
- 0
ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py View File

@@ -0,0 +1,115 @@
# Copyright 2014-2015 Red Hat, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
import requests

from ceilometer.polling.discovery.endpoint import EndpointDiscovery
from ceilometer.polling.discovery.non_openstack_credentials_discovery import \
NonOpenStackCredentialsDiscovery
from oslotest import base


class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):

class FakeResponse(object):
status_code = None
json_object = None
_content = ""

def json(self):
return self.json_object

def raise_for_status(self):
raise requests.HTTPError("Mock HTTP error.", response=self)

class FakeManager(object):
def __init__(self, keystone_client_mock):
self._keystone = keystone_client_mock

def setUp(self):
super(TestNonOpenStackCredentialsDiscovery, self).setUp()

self.discovery = NonOpenStackCredentialsDiscovery(None)

def test_discover_no_parameters(self):
result = self.discovery.discover(None, None)

self.assertEqual(['No secrets found'], result)

result = self.discovery.discover(None, "")

self.assertEqual(['No secrets found'], result)

def test_discover_no_barbican_endpoint(self):
def discover_mock(self, type):
return []

original_discover_method = EndpointDiscovery.discover
EndpointDiscovery.discover = discover_mock

result = self.discovery.discover(None, "param")

self.assertEqual(['No secrets found'], result)

EndpointDiscovery.discover = original_discover_method

@mock.patch('keystoneclient.v2_0.client.Client')
def test_discover_error_response(self, client_mock):
def discover_mock(self, type):
return ["barbican_url"]

original_discover_method = EndpointDiscovery.discover
EndpointDiscovery.discover = discover_mock

for http_status_code in requests.status_codes._codes.keys():
if http_status_code < 400:
continue

return_value = self.FakeResponse()
return_value.status_code = http_status_code
return_value.json_object = {}

client_mock.session.get.return_value = return_value

exception = self.assertRaises(
requests.HTTPError,
self.discovery.discover,
manager=self.FakeManager(client_mock),
param="param")

self.assertEqual("Mock HTTP error.", str(exception))

EndpointDiscovery.discover = original_discover_method

@mock.patch('keystoneclient.v2_0.client.Client')
def test_discover_response_ok(self, client_mock):
def discover_mock(self, type):
return ["barbican_url"]

original_discover_method = EndpointDiscovery.discover
EndpointDiscovery.discover = discover_mock

return_value = self.FakeResponse()
return_value.status_code = requests.codes.ok
return_value.json_object = {}
return_value._content = "content"

client_mock.session.get.return_value = return_value

response = self.discovery.discover(
manager=self.FakeManager(client_mock), param="param")

self.assertEqual(["content"], response)

EndpointDiscovery.discover = original_discover_method

+ 273
- 0
ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py View File

@@ -0,0 +1,273 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Tests for ceilometer/polling/non_openstack_dynamic_pollster.py
"""
import copy
import sys

import mock
import requests

from ceilometer.declarative import DynamicPollsterDefinitionException
from ceilometer.declarative import NonOpenStackApisDynamicPollsterException
from ceilometer.polling.dynamic_pollster import DynamicPollster
from ceilometer.polling.non_openstack_dynamic_pollster\
import NonOpenStackApisDynamicPollster

from oslotest import base


class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):

class FakeResponse(object):
status_code = None
json_object = None

def json(self):
return self.json_object

def raise_for_status(self):
raise requests.HTTPError("Mock HTTP error.", response=self)

def setUp(self):
super(TestNonOpenStackApisDynamicPollster, self).setUp()
self.pollster_definition_only_required_fields = {
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
'value_attribute': "volume",
'url_path': "v1/test/endpoint/fake", 'module': "module-name",
'authentication_object': "authentication_object"}

self.pollster_definition_all_fields = {
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
'value_attribute': "volume",
'url_path': "v1/test/endpoint/fake", 'module': "module-name",
'authentication_object': "authentication_object",
'user_id_attribute': 'user_id',
'project_id_attribute': 'project_id',
'resource_id_attribute': 'id', 'barbican_secret_id': 'barbican_id',
'authentication_parameters': 'parameters'}

def test_all_fields(self):
pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_only_required_fields)

all_required = ['module', 'authentication_object', 'name',
'sample_type', 'unit', 'value_attribute',
'url_path']

all_optional = ['metadata_fields', 'skip_sample_values',
'value_mapping', 'default_value', 'metadata_mapping',
'preserve_mapped_metadata', 'user_id_attribute',
'project_id_attribute', 'resource_id_attribute',
'barbican_secret_id', 'authentication_parameters',
'response_entries_key'] + all_required

for field in all_required:
self.assertIn(field, pollster.REQUIRED_POLLSTER_FIELDS)

for field in all_optional:
self.assertIn(field, pollster.ALL_POLLSTER_FIELDS)

def test_all_required_fields_exceptions(self):
pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_only_required_fields)

for key in pollster.REQUIRED_POLLSTER_FIELDS:
pollster_definition = copy.deepcopy(
self.pollster_definition_only_required_fields)
pollster_definition.pop(key)
exception = self.assertRaises(DynamicPollsterDefinitionException,
NonOpenStackApisDynamicPollster,
pollster_definition)
self.assertEqual("Required fields ['%s'] not specified."
% key, exception.brief_message)

def test_set_default_values(self):
pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_only_required_fields)

pollster_definitions = pollster.pollster_definitions

self.assertEqual(None, pollster_definitions['user_id_attribute'])
self.assertEqual(None, pollster_definitions['project_id_attribute'])
self.assertEqual(None, pollster_definitions['resource_id_attribute'])
self.assertEqual('', pollster_definitions['barbican_secret_id'])
self.assertEqual('', pollster_definitions['authentication_parameters'])

def test_user_set_optional_parameters(self):
pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_all_fields)
pollster_definitions = pollster.pollster_definitions

self.assertEqual('user_id',
pollster_definitions['user_id_attribute'])
self.assertEqual('project_id',
pollster_definitions['project_id_attribute'])
self.assertEqual('id',
pollster_definitions['resource_id_attribute'])
self.assertEqual('barbican_id',
pollster_definitions['barbican_secret_id'])
self.assertEqual('parameters',
pollster_definitions['authentication_parameters'])

def test_default_discovery_empty_secret_id(self):
pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_only_required_fields)

self.assertEqual("barbican:", pollster.default_discovery)

def test_default_discovery_not_empty_secret_id(self):
pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_all_fields)

self.assertEqual("barbican:barbican_id", pollster.default_discovery)

@mock.patch('requests.get')
def test_internal_execute_request_get_samples_status_code_ok(
self, get_mock):
sys.modules['module-name'] = mock.MagicMock()

pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_only_required_fields)

return_value = self.FakeResponse()
return_value.status_code = requests.codes.ok
return_value.json_object = {}
return_value.reason = "Ok"

get_mock.return_value = return_value

kwargs = {'resource': "credentials"}

resp, url = pollster.internal_execute_request_get_samples(kwargs)

self.assertEqual(
self.pollster_definition_only_required_fields['url_path'], url)
self.assertEqual(return_value, resp)

@mock.patch('requests.get')
def test_internal_execute_request_get_samples_status_code_not_ok(
self, get_mock):
sys.modules['module-name'] = mock.MagicMock()

pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_only_required_fields)

for http_status_code in requests.status_codes._codes.keys():
if http_status_code >= 400:
return_value = self.FakeResponse()
return_value.status_code = http_status_code
return_value.json_object = {}
return_value.reason = requests.status_codes._codes[
http_status_code][0]

get_mock.return_value = return_value

kwargs = {'resource': "credentials"}
exception = self.assertRaises(
NonOpenStackApisDynamicPollsterException,
pollster.internal_execute_request_get_samples, kwargs)

self.assertEqual(
"NonOpenStackApisDynamicPollsterException"
" None: Error while executing request[%s]."
" Status[%s] and reason [%s]."
%
(self.pollster_definition_only_required_fields['url_path'],
http_status_code, return_value.reason), str(exception))

def test_generate_new_attributes_in_sample_attribute_key_none(self):
pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_only_required_fields)

sample = {"test": "2"}
new_key = "new-key"

pollster.generate_new_attributes_in_sample(sample, None, new_key)
pollster.generate_new_attributes_in_sample(sample, "", new_key)

self.assertNotIn(new_key, sample)

def test_generate_new_attributes_in_sample(self):
pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_only_required_fields)

sample = {"test": "2"}
new_key = "new-key"

pollster.generate_new_attributes_in_sample(sample, "test", new_key)

self.assertIn(new_key, sample)
self.assertEqual(sample["test"], sample[new_key])

def test_execute_request_get_samples_non_empty_keys(self):
sample = {'user_id_attribute': "123456789",
'project_id_attribute': "dfghyt432345t",
'resource_id_attribute': "sdfghjt543"}

def execute_request_get_samples_mock(self, **kwargs):
samples = [sample]
return samples

DynamicPollster.execute_request_get_samples =\
execute_request_get_samples_mock

self.pollster_definition_all_fields[
'user_id_attribute'] = 'user_id_attribute'
self.pollster_definition_all_fields[
'project_id_attribute'] = 'project_id_attribute'
self.pollster_definition_all_fields[
'resource_id_attribute'] = 'resource_id_attribute'

pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_all_fields)

params = {"d": "d"}
response = pollster.execute_request_get_samples(**params)

self.assertEqual(sample['user_id_attribute'],
response[0]['user_id'])
self.assertEqual(sample['project_id_attribute'],
response[0]['project_id'])
self.assertEqual(sample['resource_id_attribute'],
response[0]['id'])

def test_execute_request_get_samples_empty_keys(self):
sample = {'user_id_attribute': "123456789",
'project_id_attribute': "dfghyt432345t",
'resource_id_attribute': "sdfghjt543"}

def execute_request_get_samples_mock(self, **kwargs):
samples = [sample]
return samples

DynamicPollster.execute_request_get_samples =\
execute_request_get_samples_mock

self.pollster_definition_all_fields[
'user_id_attribute'] = None
self.pollster_definition_all_fields[
'project_id_attribute'] = None
self.pollster_definition_all_fields[
'resource_id_attribute'] = None

pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_all_fields)

params = {"d": "d"}
response = pollster.execute_request_get_samples(**params)

self.assertNotIn('user_id', response[0])
self.assertNotIn('project_id', response[0])
self.assertNotIn('id', response[0])

+ 2
- 0
ceilometer/tests/unit/publisher/test_gnocchi.py View File

@@ -618,6 +618,8 @@ class PublisherWorkflowTest(base.BaseTestCase,
expected_debug = [
mock.call('filtered project found: %s',
'a2d42c23-d518-46b6-96ab-3fba2e146859'),
mock.call('Processing sample [%s] for resource ID [%s].',
self.sample, resource_id),
]

measures_posted = False

+ 90
- 4
doc/source/admin/telemetry-dynamic-pollster.rst View File

@@ -24,14 +24,12 @@ dynamic pollster system:
fashion. This feature is "a nice" to have, but is currently not
implemented.

* non-OpenStack APIs such as RadosGW (currently in development)

* APIs that return a list of entries directly, without a first key for the
list. An example is Aodh alarm list.


The dynamic pollsters system configuration
------------------------------------------
The dynamic pollsters system configuration (for OpenStack APIs)
---------------------------------------------------------------
Each YAML file in the dynamic pollster feature can use the following
attributes to define a dynamic pollster:

@@ -243,3 +241,91 @@ desires):
metadata_mapping:
name: "display_name"
default_value: 0

The dynamic pollsters system configuration (for non-OpenStack APIs)
-------------------------------------------------------------------

The dynamic pollster system can also be used for non-OpenStack APIs.
to configure non-OpenStack APIs, one can use all but one attribute of
the Dynamic pollster system. The attribute that is not supported is
the ``endpoint_type``. The dynamic pollster system for non-OpenStack APIs
is activated automatically when one uses the configurations ``module``.

The extra parameters that are available when using the Non-OpenStack
dynamic pollster sub-subsystem are the following:

* ``module``: required parameter. It is the python module name that Ceilometer
has to load to use the authentication object when executing requests against
the API. For instance, if one wants to create a pollster to gather data from
RadosGW, he/she can use the ``awsauth`` python module.

* ``authentication_object``: mandatory parameter. The name of the class that we
can find in the ``module`` that Ceilometer will use as the authentication
object in the request. For instance, when using the ``awsauth`` python module
to gather data from RadosGW, one can use the authentication object as
``S3Auth``.

* ``authentication_parameters``: optional parameter. It is a comma separated
value that will be used to instantiate the ``authentication_object``. For
instance, if we gather data from RadosGW, and we use the ``S3Auth`` class,
the ``authentication_parameters`` can be configured as
``<rados_gw_access_key>, rados_gw_secret_key, rados_gw_host_name``.

* ``barbican_secret_id``: optional parameter. The Barbican secret ID,
from which, Ceilometer can retrieve the comma separated values of the
``authentication_parameters``.

* ``user_id_attribute``: optional parameter. The name of the attribute in the
entries that are processed from ``response_entries_key`` elements that
will be mapped to ``user_id`` attribute that is sent to Gnocchi.

* ``project_id_attribute``: optional parameter. The name of the attribute in
the entries that are processed from ``response_entries_key`` elements that
will be mapped to ``project_id`` attribute that is sent to Gnocchi.

* ``resource_id_attribute``: optional parameter. The name of the attribute
in the entries that are processed from ``response_entries_key`` elements that
will be mapped to ``id`` attribute that is sent to Gnocchi.

As follows we present an example on how to convert the hard-coded pollster
for `radosgw.api.request` metric to the dynamic pollster model:

.. code-block:: yaml

---

- name: "dynamic.radosgw.api.request"
sample_type: "gauge"
unit: "request"
value_attribute: "total.ops"
url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage"
module: "awsauth"
authentication_object: "S3Auth"
authentication_parameters: "<access_key>, <secret_key>,
<rados_gateway_server>"
user_id_attribute: "user"
project_id_attribute: "user"
resource_id_attribute: "user"
response_entries_key: "summary"

We can take that example a bit further, and instead of gathering the `total
.ops` variable, which counts for all the requests (even the unsuccessful
ones), we can use the `successful_ops`.

.. code-block:: yaml

---

- name: "dynamic.radosgw.api.request.successful_ops"
sample_type: "gauge"
unit: "request"
value_attribute: "total.successful_ops"
url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage"
module: "awsauth"
authentication_object: "S3Auth"
authentication_parameters: "<access_key>, <secret_key>,
<rados_gateway_server>"
user_id_attribute: "user"
project_id_attribute: "user"
resource_id_attribute: "user"
response_entries_key: "summary"

+ 6
- 0
releasenotes/notes/dynamic-pollster-system-for-non-openstack-apis-4e06694f223f34f3.yaml View File

@@ -0,0 +1,6 @@
---
features:
- |
Add the support for non-OpenStack APIs in the dynamic pollster system.
This extension enables operators to create pollster on the fly to handle
metrics from systems such as the RadosGW API.

+ 1
- 0
setup.cfg View File

@@ -50,6 +50,7 @@ ceilometer.discover.compute =
local_instances = ceilometer.compute.discovery:InstanceDiscovery

ceilometer.discover.central =
barbican = ceilometer.polling.discovery.non_openstack_credentials_discovery:NonOpenStackCredentialsDiscovery
endpoint = ceilometer.polling.discovery.endpoint:EndpointDiscovery
tenant = ceilometer.polling.discovery.tenant:TenantDiscovery
lb_pools = ceilometer.network.services.discovery:LBPoolsDiscovery

Loading…
Cancel
Save