Add extend_lists option to merge_yaml

This allows you to extend lists in yaml config. This is useful, for
example, in prometheus.yml, where it would be nice to be able to
extend the scrape_configs to include exporters that aren't packaged
with kolla-ansible. This would provide a mechanism to do so.

Change-Id: I7a10e363f42e8ffaae3c0d2c2a758853e2cab7e1
Related: blueprint custom-prometheus-targets

ansible/action_plugins/merge_yaml.py

@@ -30,6 +30,7 @@ except ImportError:
 
 
 from ansible import constants
+from ansible import errors as ansible_errors
 from ansible.plugins import action
 
 DOCUMENTATION = '''
@@ -50,6 +51,20 @@ options:
     default: None
     required: True
     type: str
+  extend_lists:
+    description:
+      - For a given key referencing a list, this determines whether
+        the list items should be combined with the items in another
+        document if an equivalent key is found. An equivalent key
+        has the same parents and value as the first. The default
+        behaviour is to replace existing entries i.e if you have
+        two yaml documents that both define a list with an equivalent
+        key, the value from the document that appears later in the
+        list of sources will replace the value that appeared in the
+        earlier one.
+    default: False
+    required: False
+    type: bool
 author: Sean Mooney
 '''
 
@@ -107,10 +122,12 @@ class ActionModule(action.ActionBase):
 
         output = {}
         sources = self._task.args.get('sources', None)
+        extend_lists = self._task.args.get('extend_lists', False)
         if not isinstance(sources, list):
             sources = [sources]
         for source in sources:
-            Utils.update_nested_conf(output, self.read_config(source))
+            Utils.update_nested_conf(
+                output, self.read_config(source), extend_lists)
 
         # restore original vars
         self._templar.set_available_variables(old_vars)
@@ -124,7 +141,7 @@ class ActionModule(action.ActionBase):
 
             new_task = self._task.copy()
             new_task.args.pop('sources', None)
-
+            new_task.args.pop('extend_lists', None)
             new_task.args.update(
                 dict(
                     src=result_file
@@ -147,10 +164,22 @@ class ActionModule(action.ActionBase):
 
 class Utils(object):
     @staticmethod
-    def update_nested_conf(conf, update):
+    def update_nested_conf(conf, update, extend_lists=False):
         for k, v in update.items():
             if isinstance(v, dict):
-                conf[k] = Utils.update_nested_conf(conf.get(k, {}), v)
+                conf[k] = Utils.update_nested_conf(
+                    conf.get(k, {}), v, extend_lists)
+            elif k in conf and isinstance(conf[k], list) and extend_lists:
+                if not isinstance(v, list):
+                    errmsg = (
+                        "Failure merging key `%(key)s` in dictionary "
+                        "`%(dictionary)s`. Expecting a list, but received: "
+                        "`%(value)s`, which is of type: `%(type)s`" % {
+                            "key": k, "dictionary": conf,
+                            "value": v, "type": type(v)}
+                    )
+                    raise ansible_errors.AnsibleModuleError(errmsg)
+                conf[k].extend(v)
             else:
                 conf[k] = v
         return conf

tests/test_merge_yaml.py

@@ -16,12 +16,12 @@
 import imp
 import os
 
+from ansible.errors import AnsibleModuleError
 from oslotest import base
 
 PROJECT_DIR = os.path.abspath(os.path.join(os. path.dirname(__file__), '../'))
 MERGE_YAML_FILE = os.path.join(PROJECT_DIR,
                                'ansible/action_plugins/merge_yaml.py')
-
 merge_yaml = imp.load_source('merge_yaml', MERGE_YAML_FILE)
 
 
@@ -126,3 +126,51 @@ class MergeYamlConfigTest(base.BaseTestCase):
             }
         }
         self.assertDictEqual(actual, expected)
+
+    def test_merge_nested_extend_lists(self):
+        initial_conf = {
+            'level0': {
+                'level1': {
+                    "mylist": ["one", "two"]
+                },
+            }
+        }
+
+        extension = {
+            'level0': {
+                'level1': {
+                    "mylist": ["three"]
+                },
+            }
+        }
+
+        actual = merge_yaml.Utils.update_nested_conf(
+            initial_conf, extension, extend_lists=True)
+        expected = {
+            'level0': {
+                'level1': {
+                    "mylist": ["one", "two", "three"]
+                },
+            }
+        }
+        self.assertDictEqual(actual, expected)
+
+    def test_merge_nested_extend_lists_mismatch_types(self):
+        initial_conf = {
+            'level0': {
+                'level1': {
+                    "mylist": ["one", "two"]
+                },
+            }
+        }
+
+        extension = {
+            'level0': {
+                'level1': {
+                    "mylist": "three"
+                },
+            }
+        }
+        with self.assertRaisesRegex(AnsibleModuleError, "Failure merging key"):
+            merge_yaml.Utils.update_nested_conf(
+                initial_conf, extension, extend_lists=True)