diff --git a/doc/source/collect_info.rst b/doc/source/collect_info.rst new file mode 100644 index 0000000..ad3a4bb --- /dev/null +++ b/doc/source/collect_info.rst @@ -0,0 +1,55 @@ +Diagnostic info collection tools +================================ + +Configuring ansible logs and plugins +------------------------------------ + +Ansible logs and plugins are configured with the preinstall role and playbook +located in the `utils/kargo` directory. + +In order to make changes to logs configuration without running the +`kargo_deploy.sh` completely, run the following Ansible command: + +.. code:: sh + + export ws=~/workspace + /usr/bin/ansible-playbook --ssh-extra-args '-o\ StrictHostKeyChecking=no' \ + -u vagrant -b --become-user=root -i ~/${ws}inventory/inventory.cfg \ + -e @${ws}kargo/inventory/group_vars/all.yml \ + -e @${ws}utils/kargo/roles/configure_logs/defaults/main.yml \ + ${ws}utils/kargo/preinstall.yml + +Note that the `ws` var should point to the actual admin workspace directory. + +Collecting diagnostic info +-------------------------- + +There is a diagnostic info helper script located in the +`/usr/local/bin/collect_logs.sh` directory. It issues commands and collects +files given in the `${ws}utils/kargo/roles/configure_logs/defaults/main.yml` +file, from all of the cluster nodes online. Results are aggregated to the +admin node in the `logs.tar.gz` tarball. + +In order to re-build the tarball with fresh info, run: + +.. code:: sh + + ADMIN_WORKSPACE=$ws \ + VARS="-e @${ws}utils/kargo/roles/configure_logs/defaults/main.yml" \ + /usr/local/bin/collect_logs.sh + +If you deploy from the vagrantfile, you may want to use its NFS mount for +the installer repo and adjust the commands above like this: + +.. code:: sh + + export ws=/home/vagrant + /usr/bin/ansible-playbook --ssh-extra-args '-o\ StrictHostKeyChecking=no' \ + -u vagrant -b --become-user=root -i ~/${ws}inventory/inventory.cfg \ + -e @${ws}kargo/inventory/group_vars/all.yml \ + -e @/vagrant/utils/kargo/roles/configure_logs/defaults/main.yml \ + /vagrant/utils/kargo/preinstall.yml + + ADMIN_WORKSPACE=$ws \ + VARS="-e @/vagrant/utils/kargo/roles/configure_logs/defaults/main.yml" \ + /usr/local/bin/collect_logs.sh diff --git a/utils/kargo/roles/configure_logs/defaults/main.yml b/utils/kargo/roles/configure_logs/defaults/main.yml index 47a6aa7..5ebad72 100644 --- a/utils/kargo/roles/configure_logs/defaults/main.yml +++ b/utils/kargo/roles/configure_logs/defaults/main.yml @@ -16,9 +16,10 @@ callback_plugins: callback_plugin_path: /usr/share/ansible/plugins/callback # Define custom diag info to collect debug: false +searchpath: "{{workspace|default('/home/vagrant/')}}" commands: - name: git_info - cmd: cat {workspace/kargo,kargo}/.git/logs/HEAD + cmd: find . -type d -name .git -execdir sh -c 'gen-gitinfos.sh global|head -12' \; - name: timedate_info cmd: timedatectl status - name: boots_info @@ -39,6 +40,8 @@ commands: cmd: ps auxf | grep -v ]$ - name: systemctl_info cmd: systemctl status + - name: systemctl_failed_info + cmd: systemctl --state=failed --no-pager - name: k8s_resolve_info cmd: host kubernetes - name: k8s_info @@ -53,13 +56,9 @@ logs: - /var/log/daemon.log - /var/log/kern.log - /etc/resolv.conf - - kargo/cluster.yml - - kargo/kargo_default_ubuntu.yaml - - kargo/kargo_default_debian.yaml - - kargo/kargo_default_common.yaml - - kargo/inventory/inventory.cfg - - workspace/kargo/cluster.yml - - workspace/kargo/kargo_default_ubuntu.yaml - - workspace/kargo/kargo_default_debian.yaml - - workspace/kargo/kargo_default_common.yaml - - workspace/kargo/inventory/inventory.cfg + - "{{searchpath}}/kargo/cluster.yml" + - "{{searchpath}}/kargo/inventory/group_vars/all.yml" + - "{{searchpath}}/inventory/inventory.cfg" + - "{{searchpath}}/inventory/kargo_default_ubuntu.yaml" + - "{{searchpath}}/inventory/kargo_default_debian.yaml" + - "{{searchpath}}/inventory/kargo_default_common.yaml" diff --git a/utils/kargo/roles/configure_logs/templates/collect_logs.sh.j2 b/utils/kargo/roles/configure_logs/templates/collect_logs.sh.j2 index 6f84287..999ea64 100644 --- a/utils/kargo/roles/configure_logs/templates/collect_logs.sh.j2 +++ b/utils/kargo/roles/configure_logs/templates/collect_logs.sh.j2 @@ -9,4 +9,4 @@ VARS=${VARS:--e ansible_ssh_user=vagrant -e human_readable_plugin=false} /usr/bin/ansible-playbook ${LOG_LEVEL} \ --ssh-extra-args "$SSH_EXTRA_ARGS" -u ${ADMIN_USER} -b \ --become-user=root -i $ADMIN_WORKSPACE/inventory/inventory.cfg \ - ${VARS} $ADMIN_WORKSPACE/kargo/scripts/collect-info.yaml + -e worskpace=$ADMIN_WORKSPACE ${VARS} $ADMIN_WORKSPACE/kargo/scripts/collect-info.yaml