From 5215572299fa3e838635d5fa9cd4fd2a64218706 Mon Sep 17 00:00:00 2001 From: Sagi Shnaidman Date: Sat, 3 Nov 2018 01:25:39 +0200 Subject: [PATCH] Run atop for monitoring deployment Use atop[1] tool to monitor the whole job process. Atop generates binary output that could be downloaded and then investigated locally. Using atop -r /path/to/atop.bin you can read the file and by pressing "t" to move 10 seconds futher or by pressing "b" to jump to specific time in job and to see what happened on host in this time. It allows to track all resources in specific time. It allows also tracking of containers separately. For more info you can visit the site[1] If atop installation fails it shouldn't fail the job, so ignore_errors is added. Currently it's for undercloud in OVB and all nodes in multinode. [1] https://www.atoptool.nl/ Change-Id: I7e17db3e376218f620a18db7ea7ca82d7578f618 Depends-On: Ibcdcfb4d8c5c94e1a06c7e635b0b6778ad318094 --- playbooks/multinode-undercloud.yml | 6 ++++++ playbooks/ovb-setup.yml | 2 +- roles/collect-logs/defaults/main.yml | 1 + roles/undercloud-setup/defaults/main.yml | 1 + roles/undercloud-setup/tasks/atop.yml | 16 ++++++++++++++++ roles/undercloud-setup/tasks/main.yml | 2 ++ 6 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 roles/undercloud-setup/tasks/atop.yml diff --git a/playbooks/multinode-undercloud.yml b/playbooks/multinode-undercloud.yml index d5b7ad01e..8738dd620 100644 --- a/playbooks/multinode-undercloud.yml +++ b/playbooks/multinode-undercloud.yml @@ -25,6 +25,12 @@ 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 become: true + + - name: Run atop on overcloud node if configured + include_role: + name: undercloud-setup + tasks_from: atop + tags: - undercloud-setup diff --git a/playbooks/ovb-setup.yml b/playbooks/ovb-setup.yml index 42642569d..8a2662f70 100644 --- a/playbooks/ovb-setup.yml +++ b/playbooks/ovb-setup.yml @@ -13,7 +13,7 @@ - name: Setup the undercloud hosts: undercloud - gather_facts: no + gather_facts: yes vars: ansible_user: root roles: diff --git a/roles/collect-logs/defaults/main.yml b/roles/collect-logs/defaults/main.yml index 4097131c7..06d863747 100644 --- a/roles/collect-logs/defaults/main.yml +++ b/roles/collect-logs/defaults/main.yml @@ -8,6 +8,7 @@ artcl_collect_list: - /var/lib/oooq-images/*/*.sh - /var/lib/pacemaker/cib/cib* - /var/lib/pacemaker/pengine/pe-input* + - /var/log/atop* - /var/log/dmesg.txt - /var/log/host_info.txt - /var/log/journal.txt diff --git a/roles/undercloud-setup/defaults/main.yml b/roles/undercloud-setup/defaults/main.yml index 9d701abfb..dcdeaa955 100644 --- a/roles/undercloud-setup/defaults/main.yml +++ b/roles/undercloud-setup/defaults/main.yml @@ -39,3 +39,4 @@ undercloud_ara_version: 0.15.0 # NOTE(rfolco): do not configure ara on undercloud until we pin ara deps to # each branch - https://bugs.launchpad.net/tripleo/+bug/1792622 undercloud_ara: false +install_atop: true diff --git a/roles/undercloud-setup/tasks/atop.yml b/roles/undercloud-setup/tasks/atop.yml new file mode 100644 index 000000000..fc53b4b01 --- /dev/null +++ b/roles/undercloud-setup/tasks/atop.yml @@ -0,0 +1,16 @@ + +- when: install_atop|bool + ignore_errors: true + become: true + block: +# Run atop in background + - name: Install atop if configured so + shell: | + source /etc/ci/mirror_info.sh + atop_location=$(repoquery --repoid=osepel --repofrompath=osepel,$NODEPOOL_EPEL_MIRROR/7/x86_64/ --location atop) + {{ ansible_pkg_mgr }} install -y $atop_location + + - name: Run atop + shell: atop -R -w /var/log/atop.bin & + async: 0 + poll: 0 diff --git a/roles/undercloud-setup/tasks/main.yml b/roles/undercloud-setup/tasks/main.yml index ba35a8bb8..5a6e7745a 100644 --- a/roles/undercloud-setup/tasks/main.yml +++ b/roles/undercloud-setup/tasks/main.yml @@ -1,5 +1,7 @@ --- # tasks file for undercloud-setup +- name: Run atop for monitoring + include: atop.yml # Add a non-root user - name: Including non_root_user_setup tasks list