Make collection commands configurable

Allows further extension of the collection commands
* use a dictionary to define them, and their names
* making the output logging implicit (DRY)

No behavior change is expected from this change, same output files and
content is expected.

Change-Id: Ib243b0d3345faa57d9bf2e034b14e332b0e8cb80
Task: https://tree.taiga.io/project/tripleo-ci-board/task/1528

README.rst

@@ -27,8 +27,8 @@ exposed with
 Role Variables
 --------------
 
-Collection related
-~~~~~~~~~~~~~~~~~~
+File Collection
+~~~~~~~~~~~~~~~
 
 -  ``artcl_collect_list`` – A list of files and directories to gather
    from the target. Directories are collected recursively and need to
@@ -70,6 +70,34 @@ Collection related
    search, default value is 256. Note: this variable is applied only when
    ``artcl_rsync_collect_list`` is set to false.
 
+-  ``artcl_commands_extras`` - A nested dictionary of additional commands to be
+   run during collection. First level contains the group type, as defined by
+   ``collect_log_types`` list which determines which groups are collected and
+   which ones are skipped.
+
+   Defined keys will override implicit ones from defaults
+   ``artcl_commands`` which is not expected to be changed by user.
+
+   Second level keys are used to uniqly identify a command and determine the
+   default output filename, unless is mentioned via ``capture_file`` property.
+
+   ``cmd`` contains the shell command that would be run.
+
+.. code:: yaml
+
+   artcl_commands_extras:
+     system:
+       disk-space:
+         cmd: df
+         # will save output to /var/log/extras/dist-space.log
+       mounts:
+         cmd: mount -a
+         capture_file: /mounts.txt  # <-- custom capture file location
+     openstack:
+       key2:
+         cmd: touch /foo.txt
+         capture_disable: true # <-- disable implicit std redirection
+
 Documentation generation related
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

defaults/main.yml

@@ -162,6 +162,26 @@ artcl_zuul_swift_upload_path: /usr/local/bin
 artcl_collect_sosreport: false
 artcl_sosreport_options: "--batch"
 
+# User defined commands to be executed, combined with default ones.
+artcl_commands_extras: {}
+
+# Implicit commands executed by the role. Keep the dict sorted.
+artcl_commands:
+  system:
+    cpuinfo:
+      cmd: cat /proc/cpuinfo
+      capture_file: /var/log/extra/cpuinfo.txt
+    meminfo:
+      cmd: cat /proc/meminfo
+      capture_file: /var/log/extra/meminfo.txt
+    swaps:
+      cmd: cat /proc/swaps
+      capture_file: /var/log/extra/swaps.txt
+  monitoring: {}
+  network: {}
+  openstack: {}
+  container: {}
+
 # Doc generation specific vars
 artcl_gen_docs: false
 artcl_create_docs_payload:

filter_plugins/flatten_nested_dict.py

@@ -0,0 +1,16 @@
+#!/usr/bin/python
+
+
+def flatten_nested_dict(v):
+    r = []
+    for group, commands in v.items():
+        for cmd_name, cmd_dict in commands.items():
+            cmd_dict['name'] = cmd_name
+            cmd_dict['group'] = group
+            r.append(cmd_dict)
+    return r
+
+
+class FilterModule(object):
+    def filters(self):
+        return {'flatten_nested_dict': flatten_nested_dict}

molecule/default/converge.yml

@@ -26,6 +26,27 @@
       include_role:
         name: "ansible-role-collect-logs"
 
+    - name: Verify expected combined commands
+      vars:
+        expected:
+          - 'cmd': 'cat /proc/cpuinfo'
+            'capture_file': '/var/log/extra/cpuinfo.txt'
+            'name': 'cpuinfo'
+            'group': 'system'
+          - 'cmd': 'cat /proc/meminfo'
+            'capture_file': '/var/log/extra/meminfo.txt'
+            'name': 'meminfo'
+            'group': 'system'
+          - 'cmd': 'cat /proc/swaps'
+            'capture_file': '/var/log/extra/swaps.txt'
+            'name': 'swaps'
+            'group': 'system'
+      assert:
+        that: artcl_commands_flatten == expected
+        fail_msg: |
+          artcl_commands_flatten had unexpected value {{ artcl_commands_flatten }}
+        success_msg: artcl_commands_flatten had correct value
+
 - name: "Converge publish play"
   hosts: localhost
   tasks:

setup.cfg

@@ -25,6 +25,7 @@ data_files =
     usr/local/share/ansible/roles/collect-logs/tasks = tasks/*
     usr/local/share/ansible/roles/collect-logs/templates = templates/*
     usr/local/share/ansible/roles/collect-logs/files = files/*
+    usr/local/share/ansible/roles/collect-logs/filter_plugins = filter_plugins/*
     usr/local/share/ansible/roles/collect-logs/library = library/*
     usr/local/share/ansible/roles/collect-logs/module_utils = module_utils/*
     usr/local/share/ansible/roles/collect-logs/vars = vars/*

tasks/collect.yml

@@ -17,6 +17,35 @@
         src: "odl_extra_logs.j2"
         dest: "/tmp/odl_extra_logs.sh"
 
+    - name: Determine commands to run
+      # combines default dictionary with user defined one
+      # keeps only commands from groups mentioned in collect_log_types
+      run_once: true
+      vars:
+        combined_cmds: "{{ artcl_commands | combine(artcl_commands_extras, recursive=True) }}"
+      set_fact:
+        artcl_commands_flatten: '{{ combined_cmds | dict2items|selectattr("key", "in", collect_log_types) | list | items2dict | flatten_nested_dict }}'
+
+    - name: Run artcl_commands
+      # noqa 305
+      vars:
+        capture_file: |
+          {{ item.capture_file | default( item.name + '.txt') }}
+      shell:
+        # redirection of output to log file, see https://ops.tips/gists/redirect-all-outputs-of-a-bash-script-to-a-file/
+        cmd: |
+          {% if not item.capture_disable | default(False) %}
+          exec >{% if not capture_file.startswith('/') %}/var/log/extra/{% endif %}{{ capture_file }} 2>&1
+          {% endif %}
+          {# do not put anything after the command #}
+          {{ item.cmd }}
+        warn: false
+        executable: bash
+      changed_when: false
+      loop: "{{ artcl_commands_flatten }}"
+      loop_control:
+        label: "{{ item.name }}"
+
 # Change the collect_log_types if you don't want to collect
 # some specific logs
 - import_tasks: collect/system.yml

tasks/collect/system.yml

@@ -14,13 +14,6 @@
     - name: get current selinux status
       shell: "/usr/sbin/getenforce >/var/log/extra/selinux.txt"
 
-    - name: Collecting /proc/cpuinfo|meminfo|swaps
-      shell: "cat /proc/{{ item }} &> /var/log/extra/{{ item }}.txt"
-      with_items:
-        - cpuinfo
-        - meminfo
-        - swaps
-
     - name: Collect installed cron jobs
       shell: |
         for user in $(cut -f1 -d':' /etc/passwd); do \