opendev/system-config
Code Issues Proposed changes

run_cloud_launcher.sh : generate runtime stats

Browse Source 
Similar to run_all.sh (I299c0ab5dc3dea4841e560d8fb95b8f3e7df89f2),
produce a runtime stat for each run of the cloud launcher.

Although it won't directly highlight errors, problems tend to end this
playbook early.  When graphed with grafana, we could have noticed a
large drop in the average runtime which would have suggested a
problem.

Change-Id: I8e5371cbc94e9a803ea5e64ae94aca293b834c73
This commit is contained in:
Ian Wienand 2018-11-08 08:39:05 +11:00
parent 8c7e21ea9d
commit e32a9fdd0f
2 changed files with 28 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
playbooks/roles/cloud-launcher-cron/tasks/main.yaml
View File

@ -7,7 +7,7 @@
cron:
name: run_cloud_launcher.sh
state: present
job: '/usr/bin/flock -n /var/run/ansible/run_cloud_launcher.lock /bin/bash /opt/system-config/run_cloud_launcher.sh >> /var/log/ansible/run_cloud_launcher_cron.log 2>&1'
job: '/usr/bin/flock -n /var/run/ansible/run_cloud_launcher.lock /bin/bash /opt/system-config/run_cloud_launcher.sh -c >> /var/log/ansible/run_cloud_launcher_cron.log 2>&1'
minute: "{{ cloud_launcher_cron_interval.minute }}"
hour: "{{ cloud_launcher_cron_interval.hour }}"
day: "{{ cloud_launcher_cron_interval.day }}"

27
run_cloud_launcher.sh
View File

@ -27,6 +27,21 @@ ANSIBLE_PLAYBOOKS=$SYSTEM_CONFIG/playbooks
# errexit
set +e
# We only send stats if running under cron
UNDER_CRON=0
while getopts ":c" arg; do
case $arg in
c)
UNDER_CRON=1
;;
esac
done
_START_TIME=$(date '+%s')
echo "--- begin run @ $(date -Is) ---"
# We need access to all-clouds
export OS_CLIENT_CONFIG_FILE=/etc/openstack/all-clouds.yaml
@ -35,3 +50,15 @@ export OS_CLIENT_CONFIG_FILE=/etc/openstack/all-clouds.yaml
/usr/bin/timeout -k 2m 120m /usr/local/bin/ansible-playbook -i /dev/null -f 1 \
${ANSIBLE_PLAYBOOKS}/run_cloud_launcher.yaml \
-e@${ANSIBLE_PLAYBOOKS}/clouds_layouts.yml
echo "--- end run @ $(date -Is) ---"
# Only send stats under cron conditions
if [[ ${UNDER_CRON} != 1 ]]; then
return
fi
_CURRENT_TIME=$(date '+%s')
_ELAPSED_MS=$(( (_CURRENT_TIME - _START_TIME) * 1000 ))
echo "bridge.ansible.run_cloud_launcher:${_ELAPSED_MS}|ms" | \
nc -w 1 -u graphite.openstack.org 8125