Merge "Dynamic pollster system to support non-OpenStack APIs"

ceilometer/declarative.py

@@ -24,7 +24,7 @@ LOG = log.getLogger(__name__)
 
 
 class DefinitionException(Exception):
-    def __init__(self, message, definition_cfg):
+    def __init__(self, message, definition_cfg=None):
         msg = '%s %s: %s' % (self.__class__.__name__, definition_cfg, message)
         super(DefinitionException, self).__init__(msg)
         self.brief_message = message
@@ -46,6 +46,10 @@ class DynamicPollsterDefinitionException(DefinitionException):
     pass
 
 
+class NonOpenStackApisDynamicPollsterException(DefinitionException):
+    pass
+
+
 class Definition(object):
     JSONPATH_RW_PARSER = parser.ExtentedJsonPathParser()
     GETTERS_CACHE = {}

ceilometer/polling/discovery/non_openstack_credentials_discovery.py

@@ -0,0 +1,59 @@
+# Copyright 2014-2015 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log
+
+from ceilometer.polling.discovery.endpoint import EndpointDiscovery
+
+import six.moves.urllib.parse as urlparse
+
+import requests
+
+LOG = log.getLogger(__name__)
+
+
+class NonOpenStackCredentialsDiscovery(EndpointDiscovery):
+    """Barbican secrets discovery
+
+    Discovery that supplies non-OpenStack credentials for the dynamic
+    pollster sub-system. This solution uses the EndpointDiscovery to
+    find the Barbican URL where we can retrieve the credentials.
+    """
+
+    BARBICAN_URL_GET_PAYLOAD_PATTERN = "/v1/secrets/%s/payload"
+
+    def discover(self, manager, param=None):
+        barbican_secret = "No secrets found"
+        if not param:
+            return [barbican_secret]
+        barbican_endpoints = super(NonOpenStackCredentialsDiscovery,
+                                   self).discover("key-manager")
+        if not barbican_endpoints:
+            LOG.warning("No Barbican endpoints found to execute the"
+                        " credentials discovery process to [%s].",
+                        param)
+            return [barbican_secret]
+        else:
+            LOG.debug("Barbican endpoint found [%s].", barbican_endpoints)
+
+        barbican_server = next(iter(barbican_endpoints))
+        barbican_endpoint = self.BARBICAN_URL_GET_PAYLOAD_PATTERN % param
+        babrican_url = urlparse.urljoin(barbican_server, barbican_endpoint)
+
+        LOG.debug("Retrieving secrets from: %s.", babrican_url)
+        resp = manager._keystone.session.get(babrican_url, authenticated=True)
+        if resp.status_code != requests.codes.ok:
+            resp.raise_for_status()
+
+        return [resp._content]

ceilometer/polling/dynamic_pollster.py

@@ -40,20 +40,23 @@ class DynamicPollster(plugin_base.PollsterBase):
     OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values',
                                 'value_mapping', 'default_value',
                                 'metadata_mapping',
-                                'preserve_mapped_metadata'
+                                'preserve_mapped_metadata',
                                 'response_entries_key']
 
     REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit',
                                 'value_attribute', 'endpoint_type',
                                 'url_path']
 
-    ALL_POLLSTER_FIELDS = OPTIONAL_POLLSTER_FIELDS + REQUIRED_POLLSTER_FIELDS
+    # Mandatory name field
-
     name = ""
 
     def __init__(self, pollster_definitions, conf=None):
         super(DynamicPollster, self).__init__(conf)
-        LOG.debug("Dynamic pollster created with [%s]",
+
+        self.ALL_POLLSTER_FIELDS =\
+            self.OPTIONAL_POLLSTER_FIELDS + self.REQUIRED_POLLSTER_FIELDS
+
+        LOG.debug("%s instantiated with [%s]", __name__,
                   pollster_definitions)
 
         self.pollster_definitions = pollster_definitions
@@ -63,9 +66,12 @@ class DynamicPollster(plugin_base.PollsterBase):
             LOG.debug("Metadata fields configured to [%s].",
                       self.pollster_definitions['metadata_fields'])
 
+        self.set_default_values()
+
         self.name = self.pollster_definitions['name']
         self.obj = self
 
+    def set_default_values(self):
         if 'skip_sample_values' not in self.pollster_definitions:
             self.pollster_definitions['skip_sample_values'] = []
 
@@ -113,17 +119,17 @@ class DynamicPollster(plugin_base.PollsterBase):
             LOG.debug("No resources received for processing.")
             yield None
 
-        for endpoint in resources:
+        for r in resources:
-            LOG.debug("Executing get sample on URL [%s].", endpoint)
+            LOG.debug("Executing get sample for resource [%s].", r)
 
             samples = list([])
             try:
                 samples = self.execute_request_get_samples(
-                    keystone_client=manager._keystone, endpoint=endpoint)
+                    keystone_client=manager._keystone, resource=r)
             except RequestException as e:
                 LOG.warning("Error [%s] while loading samples for [%s] "
                             "for dynamic pollster [%s].",
-                            e, endpoint, self.name)
+                            e, r, self.name)
 
             for pollster_sample in samples:
                 response_value_attribute_name = self.pollster_definitions[
@@ -149,6 +155,10 @@ class DynamicPollster(plugin_base.PollsterBase):
                 if 'project_id' in pollster_sample:
                     project_id = pollster_sample["project_id"]
 
+                resource_id = None
+                if 'id' in pollster_sample:
+                    resource_id = pollster_sample["id"]
+
                 metadata = []
                 if 'metadata_fields' in self.pollster_definitions:
                     metadata = dict((k, pollster_sample.get(k))
@@ -165,7 +175,7 @@ class DynamicPollster(plugin_base.PollsterBase):
 
                     user_id=user_id,
                     project_id=project_id,
-                    resource_id=pollster_sample["id"],
+                    resource_id=resource_id,
 
                     resource_metadata=metadata
                 )
@@ -213,12 +223,8 @@ class DynamicPollster(plugin_base.PollsterBase):
     def default_discovery(self):
         return 'endpoint:' + self.pollster_definitions['endpoint_type']
 
-    def execute_request_get_samples(self, keystone_client, endpoint):
+    def execute_request_get_samples(self, **kwargs):
-        url = url_parse.urljoin(
+        resp, url = self.internal_execute_request_get_samples(kwargs)
-            endpoint, self.pollster_definitions['url_path'])
-        resp = keystone_client.session.get(url, authenticated=True)
-        if resp.status_code != requests.codes.ok:
-            resp.raise_for_status()
 
         response_json = resp.json()
 
@@ -231,6 +237,16 @@ class DynamicPollster(plugin_base.PollsterBase):
             return self.retrieve_entries_from_response(response_json)
         return []
 
+    def internal_execute_request_get_samples(self, kwargs):
+        keystone_client = kwargs['keystone_client']
+        endpoint = kwargs['resource']
+        url = url_parse.urljoin(
+            endpoint, self.pollster_definitions['url_path'])
+        resp = keystone_client.session.get(url, authenticated=True)
+        if resp.status_code != requests.codes.ok:
+            resp.raise_for_status()
+        return resp, url
+
     def retrieve_entries_from_response(self, response_json):
         if isinstance(response_json, list):
             return response_json

ceilometer/polling/manager.py

@@ -40,6 +40,7 @@ from ceilometer import declarative
 from ceilometer import keystone_client
 from ceilometer import messaging
 from ceilometer.polling import dynamic_pollster
+from ceilometer.polling import non_openstack_dynamic_pollster
 from ceilometer.polling import plugin_base
 from ceilometer.publisher import utils as publisher_utils
 from ceilometer import utils
@@ -338,10 +339,8 @@ class AgentManager(cotyledon.Service):
                     LOG.info("Loading dynamic pollster [%s] from file [%s].",
                              pollster_name, pollsters_definitions_file)
                     try:
-                        dynamic_pollster_object = dynamic_pollster.\
+                        pollsters_definitions[pollster_name] =\
-                            DynamicPollster(pollster_cfg, self.conf)
+                            self.instantiate_dynamic_pollster(pollster_cfg)
-                        pollsters_definitions[pollster_name] = \
-                            dynamic_pollster_object
                     except Exception as e:
                         LOG.error(
                             "Error [%s] while loading dynamic pollster [%s].",
@@ -356,6 +355,13 @@ class AgentManager(cotyledon.Service):
                   len(pollsters_definitions))
         return pollsters_definitions.values()
 
+    def instantiate_dynamic_pollster(self, pollster_cfg):
+        if 'module' in pollster_cfg:
+            return non_openstack_dynamic_pollster\
+                .NonOpenStackApisDynamicPollster(pollster_cfg, self.conf)
+        else:
+            return dynamic_pollster.DynamicPollster(pollster_cfg, self.conf)
+
     @staticmethod
     def _get_ext_mgr(namespace, *args, **kwargs):
         def _catch_extension_load_error(mgr, ep, exc):

ceilometer/polling/non_openstack_dynamic_pollster.py

@@ -0,0 +1,144 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+"""Non-OpenStack Dynamic pollster component
+    This component enables operators to create pollsters on the fly
+    via configuration for non-OpenStack APIs. This appraoch is quite
+    useful when adding metrics from APIs such as RadosGW into the Cloud
+    rating and billing modules.
+"""
+import copy
+import requests
+
+from ceilometer.declarative import NonOpenStackApisDynamicPollsterException
+from ceilometer.polling.dynamic_pollster import DynamicPollster
+from oslo_log import log
+
+LOG = log.getLogger(__name__)
+
+
+class NonOpenStackApisDynamicPollster(DynamicPollster):
+
+    POLLSTER_REQUIRED_POLLSTER_FIELDS = ['module', 'authentication_object']
+
+    POLLSTER_OPTIONAL_POLLSTER_FIELDS = ['user_id_attribute',
+                                         'project_id_attribute',
+                                         'resource_id_attribute',
+                                         'barbican_secret_id',
+                                         'authentication_parameters'
+                                         ]
+
+    def __init__(self, pollster_definitions, conf=None):
+        # Making sure that we do not change anything in parent classes
+        self.REQUIRED_POLLSTER_FIELDS = copy.deepcopy(
+            DynamicPollster.REQUIRED_POLLSTER_FIELDS)
+        self.OPTIONAL_POLLSTER_FIELDS = copy.deepcopy(
+            DynamicPollster.OPTIONAL_POLLSTER_FIELDS)
+
+        # Non-OpenStack dynamic pollster do not need the 'endpoint_type'.
+        self.REQUIRED_POLLSTER_FIELDS.remove('endpoint_type')
+
+        self.REQUIRED_POLLSTER_FIELDS += self.POLLSTER_REQUIRED_POLLSTER_FIELDS
+        self.OPTIONAL_POLLSTER_FIELDS += self.POLLSTER_OPTIONAL_POLLSTER_FIELDS
+
+        super(NonOpenStackApisDynamicPollster, self).__init__(
+            pollster_definitions, conf)
+
+    def set_default_values(self):
+        super(NonOpenStackApisDynamicPollster, self).set_default_values()
+
+        if 'user_id_attribute' not in self.pollster_definitions:
+            self.pollster_definitions['user_id_attribute'] = None
+
+        if 'project_id_attribute' not in self.pollster_definitions:
+            self.pollster_definitions['project_id_attribute'] = None
+
+        if 'resource_id_attribute' not in self.pollster_definitions:
+            self.pollster_definitions['resource_id_attribute'] = None
+
+        if 'barbican_secret_id' not in self.pollster_definitions:
+            self.pollster_definitions['barbican_secret_id'] = ""
+
+        if 'authentication_parameters' not in self.pollster_definitions:
+            self.pollster_definitions['authentication_parameters'] = ""
+
+    @property
+    def default_discovery(self):
+        return 'barbican:' + self.pollster_definitions['barbican_secret_id']
+
+    def internal_execute_request_get_samples(self, kwargs):
+        credentials = kwargs['resource']
+
+        override_credentials = self.pollster_definitions[
+            'authentication_parameters']
+        if override_credentials:
+            credentials = override_credentials
+
+        url = self.pollster_definitions['url_path']
+
+        authenticator_module_name = self.pollster_definitions['module']
+        authenticator_class_name = \
+            self.pollster_definitions['authentication_object']
+
+        imported_module = __import__(authenticator_module_name)
+        authenticator_class = getattr(imported_module,
+                                      authenticator_class_name)
+
+        authenticator_arguments = list(map(str.strip, credentials.split(",")))
+        authenticator_instance = authenticator_class(*authenticator_arguments)
+
+        resp = requests.get(
+            url,
+            auth=authenticator_instance)
+
+        if resp.status_code != requests.codes.ok:
+            raise NonOpenStackApisDynamicPollsterException(
+                "Error while executing request[%s]."
+                " Status[%s] and reason [%s]."
+                % (url, resp.status_code, resp.reason))
+
+        return resp, url
+
+    def execute_request_get_samples(self, **kwargs):
+        samples = super(NonOpenStackApisDynamicPollster,
+                        self).execute_request_get_samples(**kwargs)
+
+        if samples:
+            user_id_attribute = self.pollster_definitions[
+                'user_id_attribute']
+            project_id_attribute = self.pollster_definitions[
+                'project_id_attribute']
+            resource_id_attribute = self.pollster_definitions[
+                'resource_id_attribute']
+
+            for sample in samples:
+                self.generate_new_attributes_in_sample(
+                    sample, user_id_attribute, 'user_id')
+                self.generate_new_attributes_in_sample(
+                    sample, project_id_attribute, 'project_id')
+                self.generate_new_attributes_in_sample(
+                    sample, resource_id_attribute, 'id')
+
+        return samples
+
+    def generate_new_attributes_in_sample(
+            self, sample, attribute_key, new_attribute_key):
+        if attribute_key:
+            attribute_value = self.retrieve_attribute_nested_value(
+                sample, attribute_key)
+
+            LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].",
+                      attribute_key, attribute_value, sample)
+
+            sample[new_attribute_key] = attribute_value

ceilometer/publisher/gnocchi.py

@@ -338,9 +338,10 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
         for resource_id, samples_of_resource in resource_grouped_samples:
             # NOTE(sileht): / is forbidden by Gnocchi
             resource_id = resource_id.replace('/', '_')
-
             for sample in samples_of_resource:
                 metric_name = sample.name
+                LOG.debug("Processing sample [%s] for resource ID [%s].",
+                          sample, resource_id)
                 rd = self.metric_map.get(metric_name)
                 if rd is None:
                     if metric_name not in self._already_logged_metric_names:

ceilometer/tests/unit/polling/test_dynamic_pollster.py

@@ -11,7 +11,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-"""Tests for ceilometer/central/manager.py
+"""Tests for ceilometer/polling/dynamic_pollster.py
 """
 
 
@@ -162,7 +162,8 @@ class TestDynamicPollster(base.BaseTestCase):
         client_mock.session.get.return_value = return_value
 
         samples = pollster.execute_request_get_samples(
-            client_mock, "https://endpoint.server.name/")
+            keystone_client=client_mock,
+            resource="https://endpoint.server.name/")
 
         self.assertEqual(0, len(samples))
 
@@ -179,7 +180,8 @@ class TestDynamicPollster(base.BaseTestCase):
         client_mock.session.get.return_value = return_value
 
         samples = pollster.execute_request_get_samples(
-            client_mock, "https://endpoint.server.name/")
+            keystone_client=client_mock,
+            resource="https://endpoint.server.name/")
 
         self.assertEqual(3, len(samples))
 
@@ -196,8 +198,8 @@ class TestDynamicPollster(base.BaseTestCase):
 
         exception = self.assertRaises(requests.HTTPError,
                                       pollster.execute_request_get_samples,
-                                      client_mock,
+                                      keystone_client=client_mock,
-                                      "https://endpoint.server.name/")
+                                      resource="https://endpoint.server.name/")
         self.assertEqual("Mock HTTP error.", str(exception))
 
     def test_generate_new_metadata_fields_no_metadata_mapping(self):
@@ -325,7 +327,7 @@ class TestDynamicPollster(base.BaseTestCase):
 
         self.assertEqual(0, len(samples_list))
 
-    def fake_sample_list(self, keystone_client=None, endpoint=None):
+    def fake_sample_list(self, keystone_client=None, resource=None):
         samples_list = list()
         samples_list.append(
             {'name': "sample5", 'volume': 5, 'description': "desc-sample-5",

ceilometer/tests/unit/polling/test_manager.py

@@ -26,10 +26,14 @@ from stevedore import extension
 
 from ceilometer.compute import discovery as nova_discover
 from ceilometer.hardware import discovery
+from ceilometer.polling.dynamic_pollster import DynamicPollster
 from ceilometer.polling import manager
+from ceilometer.polling.non_openstack_dynamic_pollster import \
+    NonOpenStackApisDynamicPollster
 from ceilometer.polling import plugin_base
 from ceilometer import sample
 from ceilometer import service
+
 from ceilometer.tests import base
 
 
@@ -890,3 +894,24 @@ class TestPollingAgentPartitioned(BaseAgent):
             mock.call('static_3'),
             mock.call('static_4'),
         ], any_order=True)
+
+    def test_instantiate_dynamic_pollster_standard_pollster(self):
+        pollster_definition_only_required_fields = {
+            'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
+            'value_attribute': "volume", 'endpoint_type': "test",
+            'url_path': "v1/test/endpoint/fake"}
+        pollster = self.mgr.instantiate_dynamic_pollster(
+            pollster_definition_only_required_fields)
+
+        self.assertIsInstance(pollster, DynamicPollster)
+
+    def test_instantiate_dynamic_pollster_non_openstack_api(self):
+        pollster_definition_only_required_fields = {
+            'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
+            'value_attribute': "volume",
+            'url_path': "v1/test/endpoint/fake", 'module': "module-name",
+            'authentication_object': "authentication_object"}
+        pollster = self.mgr.instantiate_dynamic_pollster(
+            pollster_definition_only_required_fields)
+
+        self.assertIsInstance(pollster, NonOpenStackApisDynamicPollster)

ceilometer/tests/unit/polling/test_non_openstack_credentials_discovery.py

@@ -0,0 +1,115 @@
+# Copyright 2014-2015 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import mock
+import requests
+
+from ceilometer.polling.discovery.endpoint import EndpointDiscovery
+from ceilometer.polling.discovery.non_openstack_credentials_discovery import \
+    NonOpenStackCredentialsDiscovery
+from oslotest import base
+
+
+class TestNonOpenStackCredentialsDiscovery(base.BaseTestCase):
+
+    class FakeResponse(object):
+        status_code = None
+        json_object = None
+        _content = ""
+
+        def json(self):
+            return self.json_object
+
+        def raise_for_status(self):
+            raise requests.HTTPError("Mock HTTP error.", response=self)
+
+    class FakeManager(object):
+        def __init__(self, keystone_client_mock):
+            self._keystone = keystone_client_mock
+
+    def setUp(self):
+        super(TestNonOpenStackCredentialsDiscovery, self).setUp()
+
+        self.discovery = NonOpenStackCredentialsDiscovery(None)
+
+    def test_discover_no_parameters(self):
+        result = self.discovery.discover(None, None)
+
+        self.assertEqual(['No secrets found'], result)
+
+        result = self.discovery.discover(None, "")
+
+        self.assertEqual(['No secrets found'], result)
+
+    def test_discover_no_barbican_endpoint(self):
+        def discover_mock(self, type):
+            return []
+
+        original_discover_method = EndpointDiscovery.discover
+        EndpointDiscovery.discover = discover_mock
+
+        result = self.discovery.discover(None, "param")
+
+        self.assertEqual(['No secrets found'], result)
+
+        EndpointDiscovery.discover = original_discover_method
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_discover_error_response(self, client_mock):
+        def discover_mock(self, type):
+            return ["barbican_url"]
+
+        original_discover_method = EndpointDiscovery.discover
+        EndpointDiscovery.discover = discover_mock
+
+        for http_status_code in requests.status_codes._codes.keys():
+            if http_status_code < 400:
+                continue
+
+            return_value = self.FakeResponse()
+            return_value.status_code = http_status_code
+            return_value.json_object = {}
+
+            client_mock.session.get.return_value = return_value
+
+            exception = self.assertRaises(
+                requests.HTTPError,
+                self.discovery.discover,
+                manager=self.FakeManager(client_mock),
+                param="param")
+
+            self.assertEqual("Mock HTTP error.", str(exception))
+
+        EndpointDiscovery.discover = original_discover_method
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_discover_response_ok(self, client_mock):
+        def discover_mock(self, type):
+            return ["barbican_url"]
+
+        original_discover_method = EndpointDiscovery.discover
+        EndpointDiscovery.discover = discover_mock
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value.json_object = {}
+        return_value._content = "content"
+
+        client_mock.session.get.return_value = return_value
+
+        response = self.discovery.discover(
+            manager=self.FakeManager(client_mock), param="param")
+
+        self.assertEqual(["content"], response)
+
+        EndpointDiscovery.discover = original_discover_method

ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py

@@ -0,0 +1,273 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Tests for ceilometer/polling/non_openstack_dynamic_pollster.py
+"""
+import copy
+import sys
+
+import mock
+import requests
+
+from ceilometer.declarative import DynamicPollsterDefinitionException
+from ceilometer.declarative import NonOpenStackApisDynamicPollsterException
+from ceilometer.polling.dynamic_pollster import DynamicPollster
+from ceilometer.polling.non_openstack_dynamic_pollster\
+    import NonOpenStackApisDynamicPollster
+
+from oslotest import base
+
+
+class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
+
+    class FakeResponse(object):
+        status_code = None
+        json_object = None
+
+        def json(self):
+            return self.json_object
+
+        def raise_for_status(self):
+            raise requests.HTTPError("Mock HTTP error.", response=self)
+
+    def setUp(self):
+        super(TestNonOpenStackApisDynamicPollster, self).setUp()
+        self.pollster_definition_only_required_fields = {
+            'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
+            'value_attribute': "volume",
+            'url_path': "v1/test/endpoint/fake", 'module': "module-name",
+            'authentication_object': "authentication_object"}
+
+        self.pollster_definition_all_fields = {
+            'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
+            'value_attribute': "volume",
+            'url_path': "v1/test/endpoint/fake", 'module': "module-name",
+            'authentication_object': "authentication_object",
+            'user_id_attribute': 'user_id',
+            'project_id_attribute': 'project_id',
+            'resource_id_attribute': 'id', 'barbican_secret_id': 'barbican_id',
+            'authentication_parameters': 'parameters'}
+
+    def test_all_fields(self):
+        pollster = NonOpenStackApisDynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        all_required = ['module', 'authentication_object', 'name',
+                        'sample_type', 'unit', 'value_attribute',
+                        'url_path']
+
+        all_optional = ['metadata_fields', 'skip_sample_values',
+                        'value_mapping', 'default_value', 'metadata_mapping',
+                        'preserve_mapped_metadata', 'user_id_attribute',
+                        'project_id_attribute', 'resource_id_attribute',
+                        'barbican_secret_id', 'authentication_parameters',
+                        'response_entries_key'] + all_required
+
+        for field in all_required:
+            self.assertIn(field, pollster.REQUIRED_POLLSTER_FIELDS)
+
+        for field in all_optional:
+            self.assertIn(field, pollster.ALL_POLLSTER_FIELDS)
+
+    def test_all_required_fields_exceptions(self):
+        pollster = NonOpenStackApisDynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        for key in pollster.REQUIRED_POLLSTER_FIELDS:
+            pollster_definition = copy.deepcopy(
+                self.pollster_definition_only_required_fields)
+            pollster_definition.pop(key)
+            exception = self.assertRaises(DynamicPollsterDefinitionException,
+                                          NonOpenStackApisDynamicPollster,
+                                          pollster_definition)
+            self.assertEqual("Required fields ['%s'] not specified."
+                             % key, exception.brief_message)
+
+    def test_set_default_values(self):
+        pollster = NonOpenStackApisDynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        pollster_definitions = pollster.pollster_definitions
+
+        self.assertEqual(None, pollster_definitions['user_id_attribute'])
+        self.assertEqual(None, pollster_definitions['project_id_attribute'])
+        self.assertEqual(None, pollster_definitions['resource_id_attribute'])
+        self.assertEqual('', pollster_definitions['barbican_secret_id'])
+        self.assertEqual('', pollster_definitions['authentication_parameters'])
+
+    def test_user_set_optional_parameters(self):
+        pollster = NonOpenStackApisDynamicPollster(
+            self.pollster_definition_all_fields)
+        pollster_definitions = pollster.pollster_definitions
+
+        self.assertEqual('user_id',
+                         pollster_definitions['user_id_attribute'])
+        self.assertEqual('project_id',
+                         pollster_definitions['project_id_attribute'])
+        self.assertEqual('id',
+                         pollster_definitions['resource_id_attribute'])
+        self.assertEqual('barbican_id',
+                         pollster_definitions['barbican_secret_id'])
+        self.assertEqual('parameters',
+                         pollster_definitions['authentication_parameters'])
+
+    def test_default_discovery_empty_secret_id(self):
+        pollster = NonOpenStackApisDynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        self.assertEqual("barbican:", pollster.default_discovery)
+
+    def test_default_discovery_not_empty_secret_id(self):
+        pollster = NonOpenStackApisDynamicPollster(
+            self.pollster_definition_all_fields)
+
+        self.assertEqual("barbican:barbican_id", pollster.default_discovery)
+
+    @mock.patch('requests.get')
+    def test_internal_execute_request_get_samples_status_code_ok(
+            self, get_mock):
+        sys.modules['module-name'] = mock.MagicMock()
+
+        pollster = NonOpenStackApisDynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value.json_object = {}
+        return_value.reason = "Ok"
+
+        get_mock.return_value = return_value
+
+        kwargs = {'resource': "credentials"}
+
+        resp, url = pollster.internal_execute_request_get_samples(kwargs)
+
+        self.assertEqual(
+            self.pollster_definition_only_required_fields['url_path'], url)
+        self.assertEqual(return_value, resp)
+
+    @mock.patch('requests.get')
+    def test_internal_execute_request_get_samples_status_code_not_ok(
+            self, get_mock):
+        sys.modules['module-name'] = mock.MagicMock()
+
+        pollster = NonOpenStackApisDynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        for http_status_code in requests.status_codes._codes.keys():
+            if http_status_code >= 400:
+                return_value = self.FakeResponse()
+                return_value.status_code = http_status_code
+                return_value.json_object = {}
+                return_value.reason = requests.status_codes._codes[
+                    http_status_code][0]
+
+                get_mock.return_value = return_value
+
+                kwargs = {'resource': "credentials"}
+                exception = self.assertRaises(
+                    NonOpenStackApisDynamicPollsterException,
+                    pollster.internal_execute_request_get_samples, kwargs)
+
+                self.assertEqual(
+                    "NonOpenStackApisDynamicPollsterException"
+                    " None: Error while executing request[%s]."
+                    " Status[%s] and reason [%s]."
+                    %
+                    (self.pollster_definition_only_required_fields['url_path'],
+                     http_status_code, return_value.reason), str(exception))
+
+    def test_generate_new_attributes_in_sample_attribute_key_none(self):
+        pollster = NonOpenStackApisDynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        sample = {"test": "2"}
+        new_key = "new-key"
+
+        pollster.generate_new_attributes_in_sample(sample, None, new_key)
+        pollster.generate_new_attributes_in_sample(sample, "", new_key)
+
+        self.assertNotIn(new_key, sample)
+
+    def test_generate_new_attributes_in_sample(self):
+        pollster = NonOpenStackApisDynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        sample = {"test": "2"}
+        new_key = "new-key"
+
+        pollster.generate_new_attributes_in_sample(sample, "test", new_key)
+
+        self.assertIn(new_key, sample)
+        self.assertEqual(sample["test"], sample[new_key])
+
+    def test_execute_request_get_samples_non_empty_keys(self):
+        sample = {'user_id_attribute': "123456789",
+                  'project_id_attribute': "dfghyt432345t",
+                  'resource_id_attribute': "sdfghjt543"}
+
+        def execute_request_get_samples_mock(self, **kwargs):
+            samples = [sample]
+            return samples
+
+        DynamicPollster.execute_request_get_samples =\
+            execute_request_get_samples_mock
+
+        self.pollster_definition_all_fields[
+            'user_id_attribute'] = 'user_id_attribute'
+        self.pollster_definition_all_fields[
+            'project_id_attribute'] = 'project_id_attribute'
+        self.pollster_definition_all_fields[
+            'resource_id_attribute'] = 'resource_id_attribute'
+
+        pollster = NonOpenStackApisDynamicPollster(
+            self.pollster_definition_all_fields)
+
+        params = {"d": "d"}
+        response = pollster.execute_request_get_samples(**params)
+
+        self.assertEqual(sample['user_id_attribute'],
+                         response[0]['user_id'])
+        self.assertEqual(sample['project_id_attribute'],
+                         response[0]['project_id'])
+        self.assertEqual(sample['resource_id_attribute'],
+                         response[0]['id'])
+
+    def test_execute_request_get_samples_empty_keys(self):
+        sample = {'user_id_attribute': "123456789",
+                  'project_id_attribute': "dfghyt432345t",
+                  'resource_id_attribute': "sdfghjt543"}
+
+        def execute_request_get_samples_mock(self, **kwargs):
+            samples = [sample]
+            return samples
+
+        DynamicPollster.execute_request_get_samples =\
+            execute_request_get_samples_mock
+
+        self.pollster_definition_all_fields[
+            'user_id_attribute'] = None
+        self.pollster_definition_all_fields[
+            'project_id_attribute'] = None
+        self.pollster_definition_all_fields[
+            'resource_id_attribute'] = None
+
+        pollster = NonOpenStackApisDynamicPollster(
+            self.pollster_definition_all_fields)
+
+        params = {"d": "d"}
+        response = pollster.execute_request_get_samples(**params)
+
+        self.assertNotIn('user_id', response[0])
+        self.assertNotIn('project_id', response[0])
+        self.assertNotIn('id', response[0])

ceilometer/tests/unit/publisher/test_gnocchi.py

@@ -618,6 +618,8 @@ class PublisherWorkflowTest(base.BaseTestCase,
         expected_debug = [
             mock.call('filtered project found: %s',
                       'a2d42c23-d518-46b6-96ab-3fba2e146859'),
+            mock.call('Processing sample [%s] for resource ID [%s].',
+                      self.sample, resource_id),
         ]
 
         measures_posted = False

doc/source/admin/telemetry-dynamic-pollster.rst

@@ -24,14 +24,12 @@ dynamic pollster system:
    fashion. This feature is "a nice" to have, but is currently not
    implemented.
 
-*  non-OpenStack APIs such as RadosGW (currently in development)
-
 *  APIs that return a list of entries directly, without a first key for the
    list. An example is Aodh alarm list.
 
 
-The dynamic pollsters system configuration
+The dynamic pollsters system configuration (for OpenStack APIs)
-------------------------------------------
+---------------------------------------------------------------
 Each YAML file in the dynamic pollster feature can use the following
 attributes to define a dynamic pollster:
 
@@ -243,3 +241,91 @@ desires):
   metadata_mapping:
     name: "display_name"
   default_value: 0
+
+The dynamic pollsters system configuration (for non-OpenStack APIs)
+-------------------------------------------------------------------
+
+The dynamic pollster system can also be used for non-OpenStack APIs.
+to configure non-OpenStack APIs, one can use all but one attribute of
+the Dynamic pollster system. The attribute that is not supported is
+the ``endpoint_type``. The dynamic pollster system for non-OpenStack APIs
+is activated automatically when one uses the configurations ``module``.
+
+The extra parameters that are available when using the Non-OpenStack
+dynamic pollster sub-subsystem are the following:
+
+*  ``module``: required parameter. It is the python module name that Ceilometer
+   has to load to use the authentication object when executing requests against
+   the API. For instance, if one wants to create a pollster to gather data from
+   RadosGW, he/she can use the ``awsauth`` python module.
+
+* ``authentication_object``: mandatory parameter. The name of the class that we
+  can find in the ``module`` that Ceilometer will use as the authentication
+  object in the request. For instance, when using the ``awsauth`` python module
+  to gather data from RadosGW, one can use the authentication object as
+  ``S3Auth``.
+
+* ``authentication_parameters``: optional parameter. It is a comma separated
+  value that will be used to instantiate the ``authentication_object``. For
+  instance, if we gather data from RadosGW, and we use the ``S3Auth`` class,
+  the ``authentication_parameters`` can be configured as
+  ``<rados_gw_access_key>, rados_gw_secret_key, rados_gw_host_name``.
+
+* ``barbican_secret_id``: optional parameter. The Barbican secret ID,
+  from which, Ceilometer can retrieve the comma separated values of the
+  ``authentication_parameters``.
+
+* ``user_id_attribute``: optional parameter. The name of the attribute in the
+  entries that are processed from ``response_entries_key`` elements that
+  will be mapped to ``user_id`` attribute that is sent to Gnocchi.
+
+* ``project_id_attribute``: optional parameter. The name of the attribute in
+  the entries that are processed from ``response_entries_key`` elements that
+  will be mapped to ``project_id`` attribute that is sent to Gnocchi.
+
+* ``resource_id_attribute``: optional parameter. The name of the attribute
+  in the entries that are processed from ``response_entries_key`` elements that
+  will be mapped to ``id`` attribute that is sent to Gnocchi.
+
+As follows we present an example on how to convert the hard-coded pollster
+for `radosgw.api.request` metric to the dynamic pollster model:
+
+.. code-block:: yaml
+
+  ---
+
+  - name: "dynamic.radosgw.api.request"
+  sample_type: "gauge"
+  unit: "request"
+  value_attribute: "total.ops"
+  url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage"
+  module: "awsauth"
+  authentication_object: "S3Auth"
+  authentication_parameters: "<access_key>, <secret_key>,
+  <rados_gateway_server>"
+  user_id_attribute: "user"
+  project_id_attribute: "user"
+  resource_id_attribute: "user"
+  response_entries_key: "summary"
+
+We can take that example a bit further, and instead of gathering the `total
+.ops` variable, which counts for all the requests (even the unsuccessful
+ones), we can use the `successful_ops`.
+
+.. code-block:: yaml
+
+  ---
+
+  - name: "dynamic.radosgw.api.request.successful_ops"
+  sample_type: "gauge"
+  unit: "request"
+  value_attribute: "total.successful_ops"
+  url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage"
+  module: "awsauth"
+  authentication_object: "S3Auth"
+  authentication_parameters: "<access_key>, <secret_key>,
+  <rados_gateway_server>"
+  user_id_attribute: "user"
+  project_id_attribute: "user"
+  resource_id_attribute: "user"
+  response_entries_key: "summary"

releasenotes/notes/dynamic-pollster-system-for-non-openstack-apis-4e06694f223f34f3.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Add the support for non-OpenStack APIs in the dynamic pollster system.
+    This extension enables operators to create pollster on the fly to handle
+    metrics from systems such as the RadosGW API.

setup.cfg

@@ -50,6 +50,7 @@ ceilometer.discover.compute =
     local_instances = ceilometer.compute.discovery:InstanceDiscovery
 
 ceilometer.discover.central =
+    barbican = ceilometer.polling.discovery.non_openstack_credentials_discovery:NonOpenStackCredentialsDiscovery
     endpoint = ceilometer.polling.discovery.endpoint:EndpointDiscovery
     tenant = ceilometer.polling.discovery.tenant:TenantDiscovery
     lb_pools = ceilometer.network.services.discovery:LBPoolsDiscovery