Merge "Dynamic pollsters to support nested dictionary values"

ceilometer/polling/dynamic_pollster.py

@@ -17,17 +17,19 @@
     '/etc/ceilometer/pollsters.d/'. The pollster are defined in YAML files
     similar to the idea used for handling notifications.
 """
-
 from oslo_log import log
 from oslo_utils import timeutils
+
 from requests import RequestException
 
 from ceilometer import declarative
 from ceilometer.polling import plugin_base
 from ceilometer import sample
 
-
+from functools import reduce
+import operator
 import requests
+
 from six.moves.urllib import parse as url_parse
 
 LOG = log.getLogger(__name__)
@@ -126,7 +128,8 @@ class DynamicPollster(plugin_base.PollsterBase):
             for pollster_sample in samples:
                 response_value_attribute_name = self.pollster_definitions[
                     'value_attribute']
-                value = pollster_sample[response_value_attribute_name]
+                value = self.retrieve_attribute_nested_value(
+                    pollster_sample, response_value_attribute_name)
 
                 skip_sample_values = \
                     self.pollster_definitions['skip_sample_values']
@@ -239,4 +242,11 @@ class DynamicPollster(plugin_base.PollsterBase):
             except RuntimeError as e:
                 LOG.debug("Generator threw a StopIteration "
                           "and we need to catch it [%s].", e)
-        return response_json[first_entry_name]
+        return self.retrieve_attribute_nested_value(response_json,
+                                                    first_entry_name)
+
+    def retrieve_attribute_nested_value(self, json_object, attribute_key):
+        LOG.debug("Retrieving the nested keys [%s] from [%s].",
+                  attribute_key, json_object)
+        nested_keys = attribute_key.split(".")
+        return reduce(operator.getitem, nested_keys, json_object)

ceilometer/tests/unit/polling/test_dynamic_pollster.py

@@ -419,3 +419,33 @@ class TestDynamicPollster(base.BaseTestCase):
         entries = pollster.retrieve_entries_from_response(response)
 
         self.assertEqual(second_entries_from_response, entries)
+
+    def test_retrieve_attribute_nested_value_non_nested_key(self):
+        key = "key"
+        value = [{"d": 2}, {"g": {"h": "val"}}]
+
+        json_object = {"key": value}
+
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        returned_value = pollster.retrieve_attribute_nested_value(
+            json_object, key)
+
+        self.assertEqual(value, returned_value)
+
+    def test_retrieve_attribute_nested_value_nested_key(self):
+        key = "key.subKey"
+
+        value1 = [{"d": 2}, {"g": {"h": "val"}}]
+        sub_value = [{"r": 245}, {"h": {"yu": "yu"}}]
+
+        json_object = {"key": {"subKey": sub_value, "subkey2": value1}}
+
+        pollster = dynamic_pollster.DynamicPollster(
+            self.pollster_definition_only_required_fields)
+
+        returned_value = pollster.retrieve_attribute_nested_value(
+            json_object, key)
+
+        self.assertEqual(sub_value, returned_value)

doc/source/admin/telemetry-dynamic-pollster.rst

@@ -47,7 +47,9 @@ attributes to define a dynamic pollster:
    the unit or some other meaningful String value;
 
 *  ``value_attribute``: mandatory attribute; defines the attribute in the
-   JSON response from the URL of the component being polled. In our magnum
+   JSON response from the URL of the component being polled. We also accept
+   nested values dictionaries. To use a nested value one can simply use
+   ``attribute1.attribute2.<asMuchAsNeeded>.lastattribute``. In our magnum
    example, we can use ``status`` as the value attribute;
 
 *  ``endpoint_type``: mandatory field; defines the endpoint type that is
@@ -155,7 +157,8 @@ attributes to define a dynamic pollster:
    contains a list, instead of an object where one of its attributes is a list
    of entries, we use the list directly. Therefore, this option will be
    ignored when the API is returning the list/array of entries to be processed
-   directly.
+   directly. We also accept nested values dictionaries. To use a nested value
+   one can simply use ``attribute1.attribute2.<asMuchAsNeeded>.lastattribute``
 
 
 The complete YAML configuration to gather data from Magnum (that has been used