Merge "Add playbooks to setup logs and collect diag info"

utils/jenkins/kargo_deploy.sh

@@ -33,11 +33,19 @@ COMMON_DEFAULTS_YAML="kargo_default_common.yaml"
 COMMON_DEFAULTS_SRC="${BASH_SOURCE%/*}/../kargo/${COMMON_DEFAULTS_YAML}"
 OS_SPECIFIC_DEFAULTS_YAML="kargo_default_${NODE_BASE_OS}.yaml"
 OS_SPECIFIC_DEFAULTS_SRC="${BASH_SOURCE%/*}/../kargo/${OS_SPECIFIC_DEFAULTS_YAML}"
+LOG_LEVEL=${LOG_LEVEL:--v}
 
 required_ansible_version="2.1.0"
 
+function collect_info {
+    # Get diagnostic info and store it as the logs.tar.gz at the admin node
+    admin_node_command ADMIN_USER=$ADMIN_USER \
+        $ADMIN_WORKSPACE=$ADMIN_WORKSPACE \
+        $VARS="${LOGGING_DEFAULTS_OPT}" collect_logs.sh
+}
+
 function exit_gracefully {
-    exit_code=$?
+    local exit_code=$?
     set +e
     # set exit code if it is a param
     [[ -n "$1" ]] && exit_code=$1
@@ -225,6 +233,7 @@ cat $OS_SPECIFIC_DEFAULTS_SRC | admin_node_command "cat > $ADMIN_WORKSPACE/inven
 COMMON_DEFAULTS_OPT="-e @$ADMIN_WORKSPACE/inventory/${COMMON_DEFAULTS_YAML}"
 OS_SPECIFIC_DEFAULTS_OPT="-e @$ADMIN_WORKSPACE/inventory/${OS_SPECIFIC_DEFAULTS_YAML}"
 KARGO_DEFAULTS_OPT="-e @$ADMIN_WORKSPACE/kargo/inventory/group_vars/all.yml"
+LOGGING_DEFAULTS_OPT="-e @$ADMIN_WORKSPACE/utils/kargo/roles/configure_logs/defaults/main.yml"
 
 if [ -n "$CUSTOM_YAML" ]; then
     echo "Uploading custom YAML for deployment..."
@@ -263,6 +272,7 @@ until admin_node_command /usr/bin/ansible-playbook \
             (( tries-- ))
             echo "Deployment failed! Trying $tries more times..."
         else
+            collect_info
             exit_gracefully 1
         fi
 done
@@ -279,6 +289,7 @@ until admin_node_command /usr/bin/ansible-playbook \
             (( tries-- ))
             echo "Deployment failed! Trying $tries more times..."
         else
+            collect_info
             exit_gracefully 1
         fi
 done
@@ -295,6 +306,7 @@ until admin_node_command /usr/bin/ansible-playbook \
             (( tries-- ))
             echo "Deployment failed! Trying $tries more times..."
         else
+            collect_info
             exit_gracefully 1
         fi
 done

utils/kargo/preinstall.yml

@@ -3,3 +3,8 @@
   roles:
     - { role: preinstall, tags: preinstall }
 
+- hosts: localhost
+  become: true
+  gather_facts: no
+  roles:
+    - { role: configure_logs, tags: configure_logs }

utils/kargo/roles/configure_logs/defaults/main.yml

@@ -0,0 +1,65 @@
+log_path: /var/log/ansible/
+conf_file: /etc/ansible/ansible.cfg
+callback_plugins:
+  - name: human_readable_plugin
+    enabled: true
+    repo: https://gist.github.com/cd706de198c85a8255f6.git
+    rev: cd706de198c85a8255f6
+    script_path: ./
+    script_name: human_log.py
+  - name: timestamp_plugin
+    enabled: true
+    repo: https://github.com/bogdando/ansible-plugins.git
+    rev: 0.1.0
+    script_path: ./callback_plugins
+    script_name: timestamp.py
+callback_plugin_path: /usr/share/ansible/plugins/callback
+# Define custom diag info to collect
+debug: false
+commands:
+  - name: git_info
+    cmd: cat {workspace/kargo,kargo}/.git/logs/HEAD
+  - name: timedate_info
+    cmd: timedatectl status
+  - name: boots_info
+    cmd: journalctl --list-boots --utc --no-pager
+  - name: space_info
+    cmd: df -h
+  - name: kernel_info
+    cmd: uname -r
+  - name: distro_info
+    cmd: cat /etc/issue.net
+  - name: docker_info
+    cmd: docker info
+  - name: ip_info
+    cmd: ip -4 -o a
+  - name: route_info
+    cmd: ip ro
+  - name: proc_info
+    cmd: ps auxf | grep -v ]$
+  - name: systemctl_info
+    cmd: systemctl status
+  - name: k8s_resolve_info
+    cmd: host kubernetes
+  - name: k8s_info
+    cmd: kubectl get all --all-namespaces -o wide
+  - name: k8s_dump_info
+    cmd: kubectl get all --all-namespaces -o yaml
+  - name: errors_info
+    cmd: journalctl -p err --utc --no-pager
+logs:
+  - /var/log/ansible/ansible.log
+  - /var/log/syslog
+  - /var/log/daemon.log
+  - /var/log/kern.log
+  - /etc/resolv.conf
+  - kargo/cluster.yml
+  - kargo/kargo_default_ubuntu.yaml
+  - kargo/kargo_default_debian.yaml
+  - kargo/kargo_default_common.yaml
+  - kargo/inventory/inventory.cfg
+  - workspace/kargo/cluster.yml
+  - workspace/kargo/kargo_default_ubuntu.yaml
+  - workspace/kargo/kargo_default_debian.yaml
+  - workspace/kargo/kargo_default_common.yaml
+  - workspace/kargo/inventory/inventory.cfg

utils/kargo/roles/configure_logs/tasks/main.yml

@@ -0,0 +1,58 @@
+---
+- name: Configure logs | ensure log path
+  file:
+    path: "{{log_path}}"
+    state: directory
+    owner: "{{ansible_ssh_user}}"
+
+- name: Configure logs | ensure plugin path
+  file:
+    path: "{{callback_plugin_path}}"
+    state: directory
+    owner: "{{ansible_ssh_user}}"
+
+- name: Configure logs | get plugins
+  git:
+    repo: "{{item.repo}}"
+    dest: "/tmp/{{item.name}}/{{item.rev}}"
+  when: "{{item.enabled == true}}"
+  with_items: "{{callback_plugins}}"
+
+- name: Configure logs | install plugins
+  copy:
+    src: "/tmp/{{item.name}}/{{item.rev}}/{{item.script_path}}/{{item.script_name}}"
+    dest: "{{callback_plugin_path}}"
+  when: "{{item.enabled == true}} "
+  with_items: "{{callback_plugins}}"
+
+- name: Configure logs | remove disabled plugins
+  file:
+    path: "{{callback_plugin_path}}/{{item.script_name}}"
+    state: absent
+  when: "{{item.enabled == false}}"
+  with_items: "{{callback_plugins}}"
+
+- name: Configure logs | teardown plugins
+  file:
+    path: "/tmp/{{item.name}}/{{item.rev}}"
+    state: absent
+  when: "{{item.enabled == true}}"
+  with_items: "{{callback_plugins}}"
+
+- name: Configure logs | config
+  lineinfile:
+    line: "log_path={{log_path}}/ansible.log"
+    regexp: "^#log_path|^log_path"
+    dest: "{{conf_file}}"
+
+- name: Configure logs | callback plugin
+  lineinfile:
+    line: "callback_plugins={{callback_plugin_path}}"
+    regexp: "^#callback_plugins|^callback_plugins"
+    dest: "{{conf_file}}"
+
+- name: Configure logs | Install script for collecting info
+  template:
+    src: collect_logs.sh.j2
+    dest: "{{ bin_dir }}/collect_logs.sh"
+    mode: a+rwx

utils/kargo/roles/configure_logs/templates/collect_logs.sh.j2

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+LOG_LEVEL=${LOG_LEVEL:--v}
+SSH_EXTRA_ARGS='-o\ StrictHostKeyChecking=no'
+ADMIN_USER=${ADMIN_USER:-vagrant}
+ADMIN_WORKSPACE=${ADMIN_WORKSPACE:-workspace}
+VARS=${VARS:--e ansible_ssh_user=vagrant -e human_readable_plugin=false}
+
+/usr/bin/ansible-playbook ${LOG_LEVEL} \
+        --ssh-extra-args "$SSH_EXTRA_ARGS" -u ${ADMIN_USER} -b \
+        --become-user=root -i $ADMIN_WORKSPACE/inventory/inventory.cfg \
+        ${VARS} $ADMIN_WORKSPACE/kargo/scripts/collect-info.yaml