Convert flatten_nested_dict filter into a module

Fixed failure to run on Zuul controller due its security protections.
Modules are not affected by the same limitations as plugins or filters.

This also adds 3 module unittests.

Change-Id: Ida2f71a5077e56bf245060268d725eecac0a3e5b
Story: https://tree.taiga.io/project/tripleo-ci-board/task/1627

.envrc

@@ -0,0 +1,3 @@
+source_up
+export ANSIBLE_LIBRARY=./library
+export PYTHONPATH=./library:$PYTHONPATH

filter_plugins/flatten_nested_dict.py

@@ -1,16 +0,0 @@
-#!/usr/bin/python
-
-
-def flatten_nested_dict(v):
-    r = []
-    for group, commands in v.items():
-        for cmd_name, cmd_dict in commands.items():
-            cmd_dict['name'] = cmd_name
-            cmd_dict['group'] = group
-            r.append(cmd_dict)
-    return r
-
-
-class FilterModule(object):
-    def filters(self):
-        return {'flatten_nested_dict': flatten_nested_dict}

library/flatten_nested_dict.py

@@ -0,0 +1,88 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+from ansible.module_utils.basic import AnsibleModule
+
+
+ANSIBLE_METADATA = {
+    'metadata_version': '0.1',
+    'status': ['preview'],
+    'supported_by': 'community'
+}
+
+DOCUMENTATION = """
+module: flatten_nested_dict
+author:
+  - "Sorin Sbarnea (@ssbarnea)"
+version_added: '2.7'
+short_description: Flattens a nested dictionary into a list
+notes: []
+description:
+  - Flattens the commands nested dictionary into a list of commands.
+options:
+  data:
+    description:
+      - Nested dictionary
+    required: True
+    type: dict
+  result:
+    description:
+      - List of commands to run.
+    type: list
+    elements: dict
+"""
+EXAMPLES = """
+- name: Determine commands to run
+  flatten_nested_dict:
+    data:
+      system:
+        cmd: df
+"""
+RETURN = """
+data:
+    description: Commands to be executed
+    returned: success
+    type: list
+    sample:
+      - 'cmd': 'df'
+        'capture_file': '/var/log/extra/df.txt'
+        'name': 'df'
+        'group': 'system'
+"""
+
+
+def main():
+    result = {'data': [], 'changed': False}
+    module = AnsibleModule(
+        argument_spec=dict(
+            data=dict(type='dict', default={}),
+        ))
+    try:
+
+        for group, commands in module.params['data'].items():
+            for cmd_name, cmd_dict in commands.items():
+                cmd_dict['name'] = cmd_name
+                cmd_dict['group'] = group
+                result['data'].append(cmd_dict)
+
+    except Exception as e:
+        module.fail_json(msg=str(e))
+
+    module.exit_json(**result)
+
+
+if __name__ == '__main__':
+    main()

molecule/default/converge.yml

@@ -42,7 +42,7 @@
             'name': 'swaps'
             'group': 'system'
       assert:
-        that: artcl_commands_flatten == expected
+        that: artcl_commands_flatten['data'] == expected
         fail_msg: |
           artcl_commands_flatten had unexpected value {{ artcl_commands_flatten }}
         success_msg: artcl_commands_flatten had correct value

tasks/collect.yml

@@ -18,13 +18,14 @@
         dest: "/tmp/odl_extra_logs.sh"
 
     - name: Determine commands to run
-      # combines default dictionary with user defined one
-      # keeps only commands from groups mentioned in collect_log_types
       run_once: true
       vars:
         combined_cmds: "{{ artcl_commands | combine(artcl_commands_extras, recursive=True) }}"
-      set_fact:
-        artcl_commands_flatten: '{{ combined_cmds | dict2items|selectattr("key", "in", collect_log_types) | list | items2dict | flatten_nested_dict }}'
+      # combines default dictionary with user defined one
+      # keeps only commands from groups mentioned in collect_log_types
+      flatten_nested_dict:
+        data: "{{ combined_cmds | dict2items|selectattr('key', 'in', collect_log_types) | list | items2dict }}"
+      register: artcl_commands_flatten
 
     - name: Run artcl_commands
       # noqa 305
@@ -42,7 +43,7 @@
         warn: false
         executable: bash
       changed_when: false
-      loop: "{{ artcl_commands_flatten }}"
+      loop: "{{ artcl_commands_flatten.data }}"
       loop_control:
         label: "{{ item.name }}"
 

test-requirements.txt

@@ -1 +1,5 @@
+mock
 pre-commit>=1.20.0 # MIT
+pytest
+pytest-mock
+pyyaml

tests/common/utils.py

@@ -0,0 +1,56 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import json
+
+from ansible.module_utils import basic
+from ansible.module_utils._text import to_bytes
+
+try:
+    from unittest.mock import patch  # Python 3
+except ImportError:
+    from mock import patch  # Python 2 needs mock package installed
+
+
+def set_module_args(**args):
+    if '_ansible_remote_tmp' not in args:
+        args['_ansible_remote_tmp'] = '/tmp'
+    if '_ansible_keep_remote_files' not in args:
+        args['_ansible_keep_remote_files'] = False
+
+    args = json.dumps({'ANSIBLE_MODULE_ARGS': args})
+    basic._ANSIBLE_ARGS = to_bytes(args)
+
+
+class AnsibleExitJson(Exception):
+    pass
+
+
+class AnsibleFailJson(Exception):
+    pass
+
+
+def exit_json(*args, **kwargs):
+    if 'changed' not in kwargs:
+        kwargs['changed'] = False
+    raise AnsibleExitJson(kwargs)
+
+
+def fail_json(*args, **kwargs):
+    kwargs['failed'] = True
+    raise AnsibleFailJson(kwargs)
+
+
+class ModuleTestCase:
+    def setup_method(self):
+        self.mock_module = patch.multiple(
+            basic.AnsibleModule, exit_json=exit_json, fail_json=fail_json,
+        )
+        self.mock_module.start()
+
+    def teardown_method(self):
+        self.mock_module.stop()
+
+
+def generate_name(test_case):
+    return test_case['name']

tests/test_flatten_nested_dict.py

@@ -0,0 +1,53 @@
+import pytest  # noqa
+import flatten_nested_dict
+from common.utils import (
+    AnsibleExitJson, AnsibleFailJson, ModuleTestCase, set_module_args,
+)
+import yaml
+
+
+SAMPLE_INPUT_1 = """
+data:
+  system:
+    cpuinfo:
+      cmd: cat /proc/cpuinfo
+      capture_file: /var/log/extra/cpuinfo.txt
+"""
+
+SAMPLE_OUTPUT_1 = """
+data:
+  - cmd: cat /proc/cpuinfo
+    capture_file: /var/log/extra/cpuinfo.txt
+    name: cpuinfo
+    group: system
+"""
+
+
+class TestFlattenNestedDict(ModuleTestCase):
+
+    def test_invalid_args(self):
+        set_module_args(
+            data="invalid",
+        )
+        with pytest.raises(AnsibleFailJson) as context:
+            flatten_nested_dict.main()
+        assert context.value.args[0]['failed'] is True
+        assert 'msg' in context.value.args[0]
+
+    def test_empty(self):
+        set_module_args(
+            data={},
+        )
+        with pytest.raises(AnsibleExitJson) as context:
+            flatten_nested_dict.main()
+        assert context.value.args[0] == {'data': [], 'changed': False}
+
+    def test_one(self):
+        set_module_args(
+            data=yaml.safe_load(SAMPLE_INPUT_1)['data']
+        )
+        with pytest.raises(AnsibleExitJson) as context:
+            flatten_nested_dict.main()
+        assert context.value.args[0]['changed'] is False
+        assert context.value.args[0]['data'] == \
+            yaml.safe_load(SAMPLE_OUTPUT_1)['data']

tox.ini

@@ -51,6 +51,7 @@ setenv =
     ANSIBLE_FORCE_COLOR=1
     ANSIBLE_CALLBACK_WHITELIST=profile_tasks
     ANSIBLE_SHOW_CUSTOM_STATS=1
+    PYTHONPATH={env:PYTHONPATH:}:library
 deps =
     ansi2html  # GPL (soft-dependency of pytest-html)
     ansible>=2.9