Adjust collect-info vars

- Add failed systemd units report
- Use gitinfo script from kargo to report all found repos info
- Fix search paths for custom yaml files to be collected in
  the given workspaces
- Add docs

Change-Id: I51d29cce74f22dd1e7380e1396251c889b9339ac
Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

doc/source/collect_info.rst

@@ -0,0 +1,55 @@
+Diagnostic info collection tools
+================================
+
+Configuring ansible logs and plugins
+------------------------------------
+
+Ansible logs and plugins are configured with the preinstall role and playbook
+located in the `utils/kargo` directory.
+
+In order to make changes to logs configuration without running the
+`kargo_deploy.sh` completely, run the following Ansible command:
+
+.. code:: sh
+
+      export ws=~/workspace
+      /usr/bin/ansible-playbook --ssh-extra-args '-o\ StrictHostKeyChecking=no' \
+      -u vagrant -b --become-user=root -i ~/${ws}inventory/inventory.cfg \
+      -e @${ws}kargo/inventory/group_vars/all.yml \
+      -e @${ws}utils/kargo/roles/configure_logs/defaults/main.yml \
+      ${ws}utils/kargo/preinstall.yml
+
+Note that the `ws` var should point to the actual admin workspace directory.
+
+Collecting diagnostic info
+--------------------------
+
+There is a diagnostic info helper script located in the
+`/usr/local/bin/collect_logs.sh` directory. It issues commands and collects
+files given in the `${ws}utils/kargo/roles/configure_logs/defaults/main.yml`
+file, from all of the cluster nodes online. Results are aggregated to the
+admin node in the `logs.tar.gz` tarball.
+
+In order to re-build the tarball with fresh info, run:
+
+.. code:: sh
+
+      ADMIN_WORKSPACE=$ws \
+      VARS="-e @${ws}utils/kargo/roles/configure_logs/defaults/main.yml" \
+      /usr/local/bin/collect_logs.sh
+
+If you deploy from the vagrantfile, you may want to use its NFS mount for
+the installer repo and adjust the commands above like this:
+
+.. code:: sh
+
+      export ws=/home/vagrant
+      /usr/bin/ansible-playbook --ssh-extra-args '-o\ StrictHostKeyChecking=no' \
+      -u vagrant -b --become-user=root -i ~/${ws}inventory/inventory.cfg \
+      -e @${ws}kargo/inventory/group_vars/all.yml \
+      -e @/vagrant/utils/kargo/roles/configure_logs/defaults/main.yml \
+      /vagrant/utils/kargo/preinstall.yml
+
+      ADMIN_WORKSPACE=$ws \
+      VARS="-e @/vagrant/utils/kargo/roles/configure_logs/defaults/main.yml" \
+      /usr/local/bin/collect_logs.sh

utils/kargo/roles/configure_logs/defaults/main.yml

@@ -16,9 +16,10 @@ callback_plugins:
 callback_plugin_path: /usr/share/ansible/plugins/callback
 # Define custom diag info to collect
 debug: false
+searchpath: "{{workspace|default('/home/vagrant/')}}"
 commands:
   - name: git_info
-    cmd: cat {workspace/kargo,kargo}/.git/logs/HEAD
+    cmd: find . -type d -name .git -execdir sh -c 'gen-gitinfos.sh global|head -12' \;
   - name: timedate_info
     cmd: timedatectl status
   - name: boots_info
@@ -39,6 +40,8 @@ commands:
     cmd: ps auxf | grep -v ]$
   - name: systemctl_info
     cmd: systemctl status
+  - name: systemctl_failed_info
+    cmd: systemctl --state=failed --no-pager
   - name: k8s_resolve_info
     cmd: host kubernetes
   - name: k8s_info
@@ -53,13 +56,9 @@ logs:
   - /var/log/daemon.log
   - /var/log/kern.log
   - /etc/resolv.conf
-  - kargo/cluster.yml
-  - kargo/kargo_default_ubuntu.yaml
-  - kargo/kargo_default_debian.yaml
-  - kargo/kargo_default_common.yaml
-  - kargo/inventory/inventory.cfg
-  - workspace/kargo/cluster.yml
-  - workspace/kargo/kargo_default_ubuntu.yaml
-  - workspace/kargo/kargo_default_debian.yaml
-  - workspace/kargo/kargo_default_common.yaml
-  - workspace/kargo/inventory/inventory.cfg
+  - "{{searchpath}}/kargo/cluster.yml"
+  - "{{searchpath}}/kargo/inventory/group_vars/all.yml"
+  - "{{searchpath}}/inventory/inventory.cfg"
+  - "{{searchpath}}/inventory/kargo_default_ubuntu.yaml"
+  - "{{searchpath}}/inventory/kargo_default_debian.yaml"
+  - "{{searchpath}}/inventory/kargo_default_common.yaml"

utils/kargo/roles/configure_logs/templates/collect_logs.sh.j2

@@ -9,4 +9,4 @@ VARS=${VARS:--e ansible_ssh_user=vagrant -e human_readable_plugin=false}
 /usr/bin/ansible-playbook ${LOG_LEVEL} \
         --ssh-extra-args "$SSH_EXTRA_ARGS" -u ${ADMIN_USER} -b \
         --become-user=root -i $ADMIN_WORKSPACE/inventory/inventory.cfg \
-        ${VARS} $ADMIN_WORKSPACE/kargo/scripts/collect-info.yaml
+        -e worskpace=$ADMIN_WORKSPACE ${VARS} $ADMIN_WORKSPACE/kargo/scripts/collect-info.yaml