From 62d5a4426239125dfbb068e7dd618099cab702f0 Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Fri, 5 Aug 2016 17:26:44 +0200 Subject: [PATCH] Add playbooks to setup logs and collect diag info * Collect diag info when deployment failed and store it as a tarball at the admin node * Configure ansible logs at preinstall. * Make logs human readable with timestamps, via custom callback plugins. * Define custom diag info commands/logs/files to run and collect * Add LOG_LEVEL env var for ansible, default -v Change-Id: I3e53549589fa767f62307788b5f9654a2e4c0fbf Signed-off-by: Bogdan Dobrelya Co-authored-by: Matthew Mosesohn --- utils/jenkins/kargo_deploy.sh | 14 +++- utils/kargo/preinstall.yml | 5 ++ .../roles/configure_logs/defaults/main.yml | 65 +++++++++++++++++++ .../kargo/roles/configure_logs/tasks/main.yml | 58 +++++++++++++++++ .../templates/collect_logs.sh.j2 | 12 ++++ 5 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 utils/kargo/roles/configure_logs/defaults/main.yml create mode 100644 utils/kargo/roles/configure_logs/tasks/main.yml create mode 100644 utils/kargo/roles/configure_logs/templates/collect_logs.sh.j2 diff --git a/utils/jenkins/kargo_deploy.sh b/utils/jenkins/kargo_deploy.sh index 4001133..f4bf2db 100755 --- a/utils/jenkins/kargo_deploy.sh +++ b/utils/jenkins/kargo_deploy.sh @@ -33,11 +33,19 @@ COMMON_DEFAULTS_YAML="kargo_default_common.yaml" COMMON_DEFAULTS_SRC="${BASH_SOURCE%/*}/../kargo/${COMMON_DEFAULTS_YAML}" OS_SPECIFIC_DEFAULTS_YAML="kargo_default_${NODE_BASE_OS}.yaml" OS_SPECIFIC_DEFAULTS_SRC="${BASH_SOURCE%/*}/../kargo/${OS_SPECIFIC_DEFAULTS_YAML}" +LOG_LEVEL=${LOG_LEVEL:--v} required_ansible_version="2.1.0" +function collect_info { + # Get diagnostic info and store it as the logs.tar.gz at the admin node + admin_node_command ADMIN_USER=$ADMIN_USER \ + $ADMIN_WORKSPACE=$ADMIN_WORKSPACE \ + $VARS="${LOGGING_DEFAULTS_OPT}" collect_logs.sh +} + function exit_gracefully { - exit_code=$? + local exit_code=$? set +e # set exit code if it is a param [[ -n "$1" ]] && exit_code=$1 @@ -225,6 +233,7 @@ cat $OS_SPECIFIC_DEFAULTS_SRC | admin_node_command "cat > $ADMIN_WORKSPACE/inven COMMON_DEFAULTS_OPT="-e @$ADMIN_WORKSPACE/inventory/${COMMON_DEFAULTS_YAML}" OS_SPECIFIC_DEFAULTS_OPT="-e @$ADMIN_WORKSPACE/inventory/${OS_SPECIFIC_DEFAULTS_YAML}" KARGO_DEFAULTS_OPT="-e @$ADMIN_WORKSPACE/kargo/inventory/group_vars/all.yml" +LOGGING_DEFAULTS_OPT="-e @$ADMIN_WORKSPACE/utils/kargo/roles/configure_logs/defaults/main.yml" if [ -n "$CUSTOM_YAML" ]; then echo "Uploading custom YAML for deployment..." @@ -263,6 +272,7 @@ until admin_node_command /usr/bin/ansible-playbook \ (( tries-- )) echo "Deployment failed! Trying $tries more times..." else + collect_info exit_gracefully 1 fi done @@ -279,6 +289,7 @@ until admin_node_command /usr/bin/ansible-playbook \ (( tries-- )) echo "Deployment failed! Trying $tries more times..." else + collect_info exit_gracefully 1 fi done @@ -295,6 +306,7 @@ until admin_node_command /usr/bin/ansible-playbook \ (( tries-- )) echo "Deployment failed! Trying $tries more times..." else + collect_info exit_gracefully 1 fi done diff --git a/utils/kargo/preinstall.yml b/utils/kargo/preinstall.yml index 2876fa1..15f05d6 100644 --- a/utils/kargo/preinstall.yml +++ b/utils/kargo/preinstall.yml @@ -3,3 +3,8 @@ roles: - { role: preinstall, tags: preinstall } +- hosts: localhost + become: true + gather_facts: no + roles: + - { role: configure_logs, tags: configure_logs } diff --git a/utils/kargo/roles/configure_logs/defaults/main.yml b/utils/kargo/roles/configure_logs/defaults/main.yml new file mode 100644 index 0000000..47a6aa7 --- /dev/null +++ b/utils/kargo/roles/configure_logs/defaults/main.yml @@ -0,0 +1,65 @@ +log_path: /var/log/ansible/ +conf_file: /etc/ansible/ansible.cfg +callback_plugins: + - name: human_readable_plugin + enabled: true + repo: https://gist.github.com/cd706de198c85a8255f6.git + rev: cd706de198c85a8255f6 + script_path: ./ + script_name: human_log.py + - name: timestamp_plugin + enabled: true + repo: https://github.com/bogdando/ansible-plugins.git + rev: 0.1.0 + script_path: ./callback_plugins + script_name: timestamp.py +callback_plugin_path: /usr/share/ansible/plugins/callback +# Define custom diag info to collect +debug: false +commands: + - name: git_info + cmd: cat {workspace/kargo,kargo}/.git/logs/HEAD + - name: timedate_info + cmd: timedatectl status + - name: boots_info + cmd: journalctl --list-boots --utc --no-pager + - name: space_info + cmd: df -h + - name: kernel_info + cmd: uname -r + - name: distro_info + cmd: cat /etc/issue.net + - name: docker_info + cmd: docker info + - name: ip_info + cmd: ip -4 -o a + - name: route_info + cmd: ip ro + - name: proc_info + cmd: ps auxf | grep -v ]$ + - name: systemctl_info + cmd: systemctl status + - name: k8s_resolve_info + cmd: host kubernetes + - name: k8s_info + cmd: kubectl get all --all-namespaces -o wide + - name: k8s_dump_info + cmd: kubectl get all --all-namespaces -o yaml + - name: errors_info + cmd: journalctl -p err --utc --no-pager +logs: + - /var/log/ansible/ansible.log + - /var/log/syslog + - /var/log/daemon.log + - /var/log/kern.log + - /etc/resolv.conf + - kargo/cluster.yml + - kargo/kargo_default_ubuntu.yaml + - kargo/kargo_default_debian.yaml + - kargo/kargo_default_common.yaml + - kargo/inventory/inventory.cfg + - workspace/kargo/cluster.yml + - workspace/kargo/kargo_default_ubuntu.yaml + - workspace/kargo/kargo_default_debian.yaml + - workspace/kargo/kargo_default_common.yaml + - workspace/kargo/inventory/inventory.cfg diff --git a/utils/kargo/roles/configure_logs/tasks/main.yml b/utils/kargo/roles/configure_logs/tasks/main.yml new file mode 100644 index 0000000..a859019 --- /dev/null +++ b/utils/kargo/roles/configure_logs/tasks/main.yml @@ -0,0 +1,58 @@ +--- +- name: Configure logs | ensure log path + file: + path: "{{log_path}}" + state: directory + owner: "{{ansible_ssh_user}}" + +- name: Configure logs | ensure plugin path + file: + path: "{{callback_plugin_path}}" + state: directory + owner: "{{ansible_ssh_user}}" + +- name: Configure logs | get plugins + git: + repo: "{{item.repo}}" + dest: "/tmp/{{item.name}}/{{item.rev}}" + when: "{{item.enabled == true}}" + with_items: "{{callback_plugins}}" + +- name: Configure logs | install plugins + copy: + src: "/tmp/{{item.name}}/{{item.rev}}/{{item.script_path}}/{{item.script_name}}" + dest: "{{callback_plugin_path}}" + when: "{{item.enabled == true}} " + with_items: "{{callback_plugins}}" + +- name: Configure logs | remove disabled plugins + file: + path: "{{callback_plugin_path}}/{{item.script_name}}" + state: absent + when: "{{item.enabled == false}}" + with_items: "{{callback_plugins}}" + +- name: Configure logs | teardown plugins + file: + path: "/tmp/{{item.name}}/{{item.rev}}" + state: absent + when: "{{item.enabled == true}}" + with_items: "{{callback_plugins}}" + +- name: Configure logs | config + lineinfile: + line: "log_path={{log_path}}/ansible.log" + regexp: "^#log_path|^log_path" + dest: "{{conf_file}}" + +- name: Configure logs | callback plugin + lineinfile: + line: "callback_plugins={{callback_plugin_path}}" + regexp: "^#callback_plugins|^callback_plugins" + dest: "{{conf_file}}" + +- name: Configure logs | Install script for collecting info + template: + src: collect_logs.sh.j2 + dest: "{{ bin_dir }}/collect_logs.sh" + mode: a+rwx diff --git a/utils/kargo/roles/configure_logs/templates/collect_logs.sh.j2 b/utils/kargo/roles/configure_logs/templates/collect_logs.sh.j2 new file mode 100644 index 0000000..6f84287 --- /dev/null +++ b/utils/kargo/roles/configure_logs/templates/collect_logs.sh.j2 @@ -0,0 +1,12 @@ +#!/bin/bash + +LOG_LEVEL=${LOG_LEVEL:--v} +SSH_EXTRA_ARGS='-o\ StrictHostKeyChecking=no' +ADMIN_USER=${ADMIN_USER:-vagrant} +ADMIN_WORKSPACE=${ADMIN_WORKSPACE:-workspace} +VARS=${VARS:--e ansible_ssh_user=vagrant -e human_readable_plugin=false} + +/usr/bin/ansible-playbook ${LOG_LEVEL} \ + --ssh-extra-args "$SSH_EXTRA_ARGS" -u ${ADMIN_USER} -b \ + --become-user=root -i $ADMIN_WORKSPACE/inventory/inventory.cfg \ + ${VARS} $ADMIN_WORKSPACE/kargo/scripts/collect-info.yaml