65ddba3a99
How we got here - I3e99b80e442db0cc87f8e8c9728b7697a5e4d1d3 split the log collection into a post-run job so we always collect logs, even if the main run times out. We then realised in Ic18c89ecaf144a69e82cbe9eeed2641894af71fb that the log timestamp fact doesn't persist across playbook runs and it's not totally clear how getting it from hostvars interacts with dynamic inventory. Thus take an approach that doesn't rely on passing variables; this simply pulls the time from the stamp we put on the first line of the log file. We then use that to rename the stored file, which should correspond more closely with the time the Zuul job actually started. To further remove confusion when looking at a lot of logs, reset the timestamps to this time as well. Change-Id: I7a115c75286e03b09ac3b8982ff0bd01037d34dd
43 lines
2.0 KiB
YAML
43 lines
2.0 KiB
YAML
- hosts: localhost
|
|
tasks:
|
|
- name: Add bridge.o.o to inventory for playbook
|
|
add_host:
|
|
name: bridge.openstack.org
|
|
ansible_python_interpreter: python3
|
|
ansible_user: zuul
|
|
# Without setting ansible_host directly, mirror-workspace-git-repos
|
|
# gets sad because if delegate_to localhost and with add_host that
|
|
# ends up with ansible_host being localhost.
|
|
ansible_host: bridge.openstack.org
|
|
ansible_port: 22
|
|
|
|
- hosts: bridge.openstack.org
|
|
tasks:
|
|
- name: Run the production playbook and capture logs
|
|
block:
|
|
- name: Get a current timestamp
|
|
set_fact:
|
|
_log_timestamp: "{{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%S') }}"
|
|
|
|
# NOTE(ianw) : this gets parsed by the post playbook. If this
|
|
# is updated, that parsing should be too
|
|
- name: Log a playbook start header
|
|
become: yes
|
|
shell: 'echo "Running {{ _log_timestamp }}: ansible-playbook -v -f {{ infra_prod_ansible_forks }} /home/zuul/src/opendev.org/opendev/system-config/playbooks/{{ playbook_name }}" > /var/log/ansible/{{ playbook_name }}.log'
|
|
|
|
- name: Run specified playbook on bridge.o.o and redirect output
|
|
become: yes
|
|
shell: 'ansible-playbook -v -f {{ infra_prod_ansible_forks }} /home/zuul/src/opendev.org/opendev/system-config/playbooks/{{ playbook_name }} >> /var/log/ansible/{{ playbook_name }}.log'
|
|
register: _run
|
|
|
|
always:
|
|
- name: Send run stats
|
|
shell: |
|
|
# delta is in string format h:m:s.sss; convert to ms for statsd
|
|
{% set delta = _run.delta.split(':') %}
|
|
{% set delta_ms = ((delta[0]|int * 60 * 60 * 1000) + (delta[1]|int * 60 * 1000) + (delta[2]|float * 1000)) | int %}
|
|
echo 'bridge.ansible.{{ zuul.job }}.runtime:{{ delta_ms }}|ms' | nc -w 1 -u graphite.opendev.org 8125
|
|
echo 'bridge.ansible.{{ zuul.job }}.rc:{{ _run.rc }}|g' | nc -w 1 -u graphite.opendev.org 8125
|
|
args:
|
|
executable: '/bin/bash'
|