Enable processing metadata with nested objects

Depends-On: https://review.opendev.org/#/c/709807/
Change-Id: Id108a7cd7bf356156b68324c3d033037a84869fc

ceilometer/polling/dynamic_pollster.py

@@ -111,10 +111,14 @@ class PollsterSampleExtractor(object):
     def generate_sample(self, pollster_sample, pollster_definitons=None):
         pollster_definitions =\
             pollster_definitons or self.definitions.configurations
-        metadata = []
+
+        metadata = dict()
         if 'metadata_fields' in pollster_definitions:
-            metadata = dict((k, pollster_sample.get(k))
-                            for k in pollster_definitions['metadata_fields'])
+            for k in pollster_definitions['metadata_fields']:
+                val = self.retrieve_attribute_nested_value(pollster_sample, k)
+
+                if val:
+                    metadata[k] = val
 
         self.generate_new_metadata_fields(
             metadata=metadata, pollster_definitions=pollster_definitions)
@@ -134,8 +138,12 @@ class PollsterSampleExtractor(object):
 
         attribute_key = value_attribute or self.definitions.\
             extract_attribute_key()
-        LOG.debug("Retrieving the nested keys [%s] from [%s].",
-                  attribute_key, json_object)
+
+        LOG.debug(
+            "Retrieving the nested keys [%s] from [%s] or pollster [""%s].",
+            attribute_key, json_object,
+            self.definitions.configurations["name"])
+
         keys_and_operations = attribute_key.split("|")
         attribute_key = keys_and_operations[0].strip()
 
@@ -158,11 +166,15 @@ class PollsterSampleExtractor(object):
                     "The attribute field operation [%s] must use the ["
                     "value] variable." % operation,
                     self.definitions.configurations)
-            LOG.debug("Executing operation [%s] against value [%s].",
-                      operation, value)
+
+            LOG.debug("Executing operation [%s] against value[%s] for "
+                      "pollster [%s].", operation, value,
+                      self.definitions.configurations["name"])
+
             value = eval(operation.strip())
-            LOG.debug("Result [%s] of operation [%s].",
-                      value, operation)
+            LOG.debug(
+                "Result [%s] of operation [%s] for pollster [%s].",
+                value, operation, self.definitions.configurations["name"])
         return value
 
 

ceilometer/tests/unit/polling/test_dynamic_pollster.py

@@ -847,3 +847,45 @@ class TestDynamicPollster(base.BaseTestCase):
         self.assertTrue("headers" not in request_args)
         self.assertTrue("authenticated" in request_args)
         self.assertTrue(request_args["authenticated"])
+
+    @mock.patch('keystoneclient.v2_0.client.Client')
+    def test_metadata_nested_objects(self, keystone_mock):
+        generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={
+            'flavor': [{"name": "a", "ram": 1}, {"name": "b", "ram": 2},
+                       {"name": "c", "ram": 3}, {"name": "d", "ram": 4},
+                       {"name": "e", "ram": 5}, {"name": "f", "ram": 6},
+                       {"name": "g", "ram": 7}, {"name": "h", "ram": 8}],
+            'name': ['s1', 's2', 's3', 's4', 's5', 's6', 's7', 's8'],
+            'state': ['Active', 'Error', 'Down', 'Active', 'Active',
+                      'Migrating', 'Active', 'Error']
+        }, dict_name='servers', page_link_name='server_link')
+
+        generator.generate_samples('http://test.com/v1/test-servers', {
+            'marker=c3': 3,
+            'marker=f6': 3
+        }, 2)
+
+        keystone_mock.session.get.side_effect = generator.mock_request
+        fake_manager = self.FakeManager(keystone=keystone_mock)
+
+        pollster_definition = dict(self.multi_metric_pollster_definition)
+        pollster_definition['name'] = 'test-pollster'
+        pollster_definition['value_attribute'] = 'state'
+        pollster_definition['url_path'] = 'v1/test-servers'
+        pollster_definition['response_entries_key'] = 'servers'
+        pollster_definition['metadata_fields'] = ['flavor.name', 'flavor.ram']
+        pollster_definition['next_sample_url_attribute'] = \
+            'server_link | filter(lambda v: v.get("rel") == "next", value) |' \
+            'list(value)| value [0] | value.get("href")'
+        pollster = dynamic_pollster.DynamicPollster(pollster_definition)
+
+        samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
+
+        samples = list(samples)
+
+        self.assertEqual(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'],
+                         list(map(lambda s: s.resource_metadata["flavor.name"],
+                                  samples)))
+        self.assertEqual(list(range(1, 9)),
+                         list(map(lambda s: s.resource_metadata["flavor.ram"],
+                                  samples)))

doc/source/admin/telemetry-dynamic-pollster.rst

@@ -60,7 +60,9 @@ attributes to define a dynamic pollster:
 
 *  ``metadata_fields``: optional field. It is a list of all fields that
    the response of the request executed with ``url_path`` that we want to
-   retrieve. As an example, for magnum, one can use the following values:
+   retrieve. To use a nested value one can simply use
+   ``attribute1.attribute2.<asMuchAsNeeded>.lastattribute``. As an example,
+   for magnum, one can use the following values:
 
   .. code-block:: yaml