Merge "Add response handlers to support different response types"

ceilometer/declarative.py

@@ -49,6 +49,10 @@ class DynamicPollsterDefinitionException(DynamicPollsterException):
     pass
 
 
+class InvalidResponseTypeException(DynamicPollsterException):
+    pass
+
+
 class NonOpenStackApisDynamicPollsterException\
         (DynamicPollsterDefinitionException):
     pass

ceilometer/polling/dynamic_pollster.py

@@ -18,8 +18,10 @@
     similar to the idea used for handling notifications.
 """
 import copy
+import json
 import re
 import time
+import xmltodict
 
 from oslo_log import log
 
@@ -46,6 +48,80 @@ def validate_sample_type(sample_type):
             % (sample_type, ceilometer_sample.TYPES))
 
 
+class XMLResponseHandler(object):
+    """This response handler converts an XML in string format to a dict"""
+
+    @staticmethod
+    def handle(response):
+        return xmltodict.parse(response)
+
+
+class JsonResponseHandler(object):
+    """This response handler converts a JSON in string format to a dict"""
+
+    @staticmethod
+    def handle(response):
+        return json.loads(response)
+
+
+class PlainTextResponseHandler(object):
+    """This response handler converts a string to a dict {'out'=<string>}"""
+
+    @staticmethod
+    def handle(response):
+        return {'out': str(response)}
+
+
+VALID_HANDLERS = {
+    'json': JsonResponseHandler,
+    'xml': XMLResponseHandler,
+    'text': PlainTextResponseHandler
+}
+
+
+def validate_response_handler(val):
+    if not isinstance(val, list):
+        raise declarative.DynamicPollsterDefinitionException(
+            "Invalid response_handlers configuration. It must be a list. "
+            "Provided value type: %s" % type(val).__name__)
+
+    for value in val:
+        if value not in VALID_HANDLERS:
+            raise declarative.DynamicPollsterDefinitionException(
+                "Invalid response_handler value [%s]. Accepted values "
+                "are [%s]" % (value, ', '.join(list(VALID_HANDLERS))))
+
+
+class ResponseHandlerChain(object):
+    """Tries to convert a string to a dict using the response handlers"""
+
+    def __init__(self, response_handlers, **meta):
+        if not isinstance(response_handlers, list):
+            response_handlers = list(response_handlers)
+
+        self.response_handlers = response_handlers
+        self.meta = meta
+
+    def handle(self, response):
+        failed_handlers = []
+        for handler in self.response_handlers:
+            try:
+                return handler.handle(response)
+            except Exception as e:
+                handler_name = handler.__name__
+                failed_handlers.append(handler_name)
+                LOG.debug(
+                    "Error handling response [%s] with handler [%s]: %s. "
+                    "We will try the next one, if multiple handlers were "
+                    "configured.",
+                    response, handler_name, e)
+
+        handlers_str = ', '.join(failed_handlers)
+        raise declarative.InvalidResponseTypeException(
+            "No remaining handlers to handle the response [%s], "
+            "used handlers [%s]. [%s]." % (response, handlers_str, self.meta))
+
+
 class PollsterDefinitionBuilder(object):
 
     def __init__(self, definitions):
@@ -440,7 +516,9 @@ class PollsterDefinitions(object):
         PollsterDefinition(name='timeout', default=30),
         PollsterDefinition(name='extra_metadata_fields_cache_seconds',
                            default=3600),
-        PollsterDefinition(name='extra_metadata_fields')
+        PollsterDefinition(name='extra_metadata_fields'),
+        PollsterDefinition(name='response_handlers', default=['json'],
+                           validator=validate_response_handler)
     ]
 
     extra_definitions = []
@@ -655,6 +733,11 @@ class PollsterSampleGatherer(object):
 
     def __init__(self, definitions):
         self.definitions = definitions
+        self.response_handler_chain = ResponseHandlerChain(
+            map(VALID_HANDLERS.get,
+                self.definitions.configurations['response_handlers']),
+            url_path=definitions.configurations['url_path']
+        )
 
     @property
     def default_discovery(self):
@@ -668,17 +751,17 @@ class PollsterSampleGatherer(object):
         resp, url = self._internal_execute_request_get_samples(
             definitions=definitions, **kwargs)
 
-        response_json = resp.json()
-        entry_size = len(response_json)
-        LOG.debug("Entries [%s] in the JSON for request [%s] "
+        response_dict = self.response_handler_chain.handle(resp.text)
+        entry_size = len(response_dict)
+        LOG.debug("Entries [%s] in the DICT for request [%s] "
                   "for dynamic pollster [%s].",
-                  response_json, url, definitions['name'])
+                  response_dict, url, definitions['name'])
 
         if entry_size > 0:
             samples = self.retrieve_entries_from_response(
-                response_json, definitions)
+                response_dict, definitions)
             url_to_next_sample = self.get_url_to_next_sample(
-                response_json, definitions)
+                response_dict, definitions)
 
             self.prepare_samples(definitions, samples, **kwargs)
 

ceilometer/tests/unit/polling/test_dynamic_pollster.py

@@ -14,13 +14,14 @@
 """Tests for OpenStack dynamic pollster
 """
 import copy
+import json
 import logging
 from unittest import mock
 
 import requests
 from urllib import parse as urlparse
 
-from ceilometer.declarative import DynamicPollsterDefinitionException
+from ceilometer import declarative
 from ceilometer.polling import dynamic_pollster
 from ceilometer import sample
 from oslotest import base
@@ -107,6 +108,11 @@ class TestDynamicPollster(base.BaseTestCase):
     class FakeResponse(object):
         status_code = None
         json_object = None
+        _text = None
+
+        @property
+        def text(self):
+            return self._text or json.dumps(self.json_object)
 
         def json(self):
             return self.json_object
@@ -242,9 +248,10 @@ class TestDynamicPollster(base.BaseTestCase):
             pollster_definition = copy.deepcopy(
                 self.pollster_definition_only_required_fields)
             pollster_definition.pop(key)
-            exception = self.assertRaises(DynamicPollsterDefinitionException,
-                                          dynamic_pollster.DynamicPollster,
-                                          pollster_definition)
+            exception = self.assertRaises(
+                declarative.DynamicPollsterDefinitionException,
+                dynamic_pollster.DynamicPollster,
+                pollster_definition)
             self.assertEqual("Required fields ['%s'] not specified."
                              % key, exception.brief_message)
 
@@ -252,7 +259,7 @@ class TestDynamicPollster(base.BaseTestCase):
         self.pollster_definition_only_required_fields[
             'sample_type'] = "invalid_sample_type"
         exception = self.assertRaises(
-            DynamicPollsterDefinitionException,
+            declarative.DynamicPollsterDefinitionException,
             dynamic_pollster.DynamicPollster,
             self.pollster_definition_only_required_fields)
         self.assertEqual("Invalid sample type [invalid_sample_type]. "
@@ -313,6 +320,147 @@ class TestDynamicPollster(base.BaseTestCase):
 
         self.assertEqual(3, len(samples))
 
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_json_response_handler(
+            self, client_mock):
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value._text = '{"test": [1,2,3]}'
+
+        client_mock.session.get.return_value = return_value
+
+        samples = pollster.definitions.sample_gatherer. \
+            execute_request_get_samples(
+                keystone_client=client_mock,
+                resource="https://endpoint.server.name/")
+
+        self.assertEqual(3, len(samples))
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_xml_response_handler(
+            self, client_mock):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['response_handlers'] = ['xml']
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value._text = '<test>123</test>'
+        client_mock.session.get.return_value = return_value
+
+        samples = pollster.definitions.sample_gatherer. \
+            execute_request_get_samples(
+                keystone_client=client_mock,
+                resource="https://endpoint.server.name/")
+
+        self.assertEqual(3, len(samples))
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_xml_json_response_handler(
+            self, client_mock):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['response_handlers'] = ['xml', 'json']
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value._text = '<test>123</test>'
+        client_mock.session.get.return_value = return_value
+
+        samples = pollster.definitions.sample_gatherer. \
+            execute_request_get_samples(
+                keystone_client=client_mock,
+                resource="https://endpoint.server.name/")
+
+        self.assertEqual(3, len(samples))
+
+        return_value._text = '{"test": [1,2,3,4]}'
+
+        samples = pollster.definitions.sample_gatherer. \
+            execute_request_get_samples(
+                keystone_client=client_mock,
+                resource="https://endpoint.server.name/")
+
+        self.assertEqual(4, len(samples))
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_execute_request_xml_json_response_handler_invalid_response(
+            self, client_mock):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['response_handlers'] = ['xml', 'json']
+        pollster = dynamic_pollster.DynamicPollster(definitions)
+
+        return_value = self.FakeResponse()
+        return_value.status_code = requests.codes.ok
+        return_value._text = 'Invalid response'
+        client_mock.session.get.return_value = return_value
+
+        with self.assertLogs('ceilometer.polling.dynamic_pollster',
+                             level='DEBUG') as logs:
+            gatherer = pollster.definitions.sample_gatherer
+            exception = self.assertRaises(
+                declarative.InvalidResponseTypeException,
+                gatherer.execute_request_get_samples,
+                keystone_client=client_mock,
+                resource="https://endpoint.server.name/")
+
+            xml_handling_error = logs.output[2]
+            json_handling_error = logs.output[3]
+
+            self.assertIn(
+                'DEBUG:ceilometer.polling.dynamic_pollster:'
+                'Error handling response [Invalid response] '
+                'with handler [XMLResponseHandler]',
+                xml_handling_error)
+
+            self.assertIn(
+                'DEBUG:ceilometer.polling.dynamic_pollster:'
+                'Error handling response [Invalid response] '
+                'with handler [JsonResponseHandler]',
+                json_handling_error)
+
+            self.assertEqual(
+                "InvalidResponseTypeException None: "
+                "No remaining handlers to handle the response "
+                "[Invalid response], used handlers "
+                "[XMLResponseHandler, JsonResponseHandler]. "
+                "[{'url_path': 'v1/test/endpoint/fake'}].",
+                str(exception))
+
+    def test_configure_response_handler_definition_invalid_value(self):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['response_handlers'] = ['jason']
+
+        exception = self.assertRaises(
+            declarative.DynamicPollsterDefinitionException,
+            dynamic_pollster.DynamicPollster,
+            pollster_definitions=definitions)
+        self.assertEqual("DynamicPollsterDefinitionException None: "
+                         "Invalid response_handler value [jason]. "
+                         "Accepted values are [json, xml, text]",
+                         str(exception))
+
+    def test_configure_response_handler_definition_invalid_type(self):
+        definitions = copy.deepcopy(
+            self.pollster_definition_only_required_fields)
+        definitions['response_handlers'] = 'json'
+
+        exception = self.assertRaises(
+            declarative.DynamicPollsterDefinitionException,
+            dynamic_pollster.DynamicPollster,
+            pollster_definitions=definitions)
+        self.assertEqual("DynamicPollsterDefinitionException None: "
+                         "Invalid response_handlers configuration. "
+                         "It must be a list. Provided value type: str",
+                         str(exception))
+
     @mock.patch('keystoneclient.v2_0.client.Client')
     def test_execute_request_get_samples_exception_on_request(
             self, client_mock):
@@ -728,6 +876,10 @@ class TestDynamicPollster(base.BaseTestCase):
 
         def internal_execute_request_get_samples_mock(self, **kwargs):
             class Response:
+                @property
+                def text(self):
+                    return json.dumps([sample])
+
                 def json(self):
                     return [sample]
             return Response(), "url"

ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py

@@ -15,6 +15,7 @@
 """
 
 import copy
+import json
 import sys
 from unittest import mock
 
@@ -312,6 +313,11 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
         def internal_execute_request_get_samples_mock(
                 self, definitions, **kwargs):
             class Response:
+
+                @property
+                def text(self):
+                    return json.dumps([sample])
+
                 def json(self):
                     return [sample]
             return Response(), "url"

doc/source/admin/telemetry-dynamic-pollster.rst

@@ -45,7 +45,7 @@ attributes to define a dynamic pollster:
    the unit or some other meaningful String value;
 
 *  ``value_attribute``: mandatory attribute; defines the attribute in the
-   JSON response from the URL of the component being polled. We also accept
+   response from the URL of the component being polled. We also accept
    nested values dictionaries. To use a nested value one can simply use
    ``attribute1.attribute2.<asMuchAsNeeded>.lastattribute``. It is also
    possible to reference the sample itself using ``"." (dot)``; the self
@@ -286,6 +286,117 @@ desires):
         name: "display_name"
     default_value: 0
 
+*  ``response_handlers``: optional parameter. Defines the response
+   handlers used to handle the response. For now, the supported values
+   are:
+
+   ``json``: This handler will interpret the response as a `JSON` and will
+   convert it to a `dictionary` which can be manipulated using the
+   operations options when mapping the attributes:
+
+   .. code-block:: yaml
+
+     ---
+
+     - name: "dynamic.json.response"
+       sample_type: "gauge"
+       [...]
+       response_handlers:
+          - json
+
+   Response to handle:
+
+   .. code-block:: json
+
+     {
+        "test": {
+          "list": [1, 2, 3]
+        }
+     }
+
+   Response handled:
+
+   .. code-block:: python
+
+     {
+        'test': {
+          'list': [1, 2, 3]
+        }
+     }
+
+
+   ``xml``: This handler will interpret the response as an `XML` and will
+   convert it to a `dictionary` which can be manipulated using the
+   operations options when mapping the attributes:
+
+   .. code-block:: yaml
+
+     ---
+
+     - name: "dynamic.json.response"
+       sample_type: "gauge"
+       [...]
+       response_handlers:
+          - xml
+
+   Response to handle:
+
+   .. code-block:: xml
+
+     <test>
+       <list>1</list>
+       <list>2</list>
+       <list>3</list>
+     </test>
+
+   Response handled:
+
+   .. code-block:: python
+
+     {
+        'test': {
+          'list': [1, 2, 3]
+        }
+     }
+
+   ``text``: This handler will interpret the response as a `PlainText` and
+   will convert it to a `dictionary` which can be manipulated using the
+   operations options when mapping the attributes:
+
+   .. code-block:: yaml
+
+     ---
+
+     - name: "dynamic.json.response"
+       sample_type: "gauge"
+       [...]
+       response_handlers:
+          - text
+
+   Response to handle:
+
+   .. code-block:: text
+
+     Plain text response
+
+   Response handled:
+
+   .. code-block:: python
+
+     {
+        'out': "Plain text response"
+     }
+
+   They can be used together or individually. If not defined, the
+   `default` value will be `json`. If you set 2 or more response
+   handlers, the first configured handler will be used to try to
+   handle the response, if it is not possible, a `DEBUG` log
+   message will be displayed, then the next will be used
+   and so on. If no configured handler was able to handle
+   the response, an empty dict will be returned and a `WARNING`
+   log will be displayed to warn operators that the response was
+   not able to be handled by any configured handler.
+
 The dynamic pollsters system configuration (for non-OpenStack APIs)
 -------------------------------------------------------------------
 

requirements.txt

@@ -6,6 +6,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
+xmltodict>=0.13.0 # MIT License
 cachetools>=2.1.0 # MIT License
 cotyledon>=1.3.0 #Apache-2.0
 futurist>=1.8.0 # Apache-2.0