65ddba3a99
How we got here - I3e99b80e442db0cc87f8e8c9728b7697a5e4d1d3 split the log collection into a post-run job so we always collect logs, even if the main run times out. We then realised in Ic18c89ecaf144a69e82cbe9eeed2641894af71fb that the log timestamp fact doesn't persist across playbook runs and it's not totally clear how getting it from hostvars interacts with dynamic inventory. Thus take an approach that doesn't rely on passing variables; this simply pulls the time from the stamp we put on the first line of the log file. We then use that to rename the stored file, which should correspond more closely with the time the Zuul job actually started. To further remove confusion when looking at a lot of logs, reset the timestamps to this time as well. Change-Id: I7a115c75286e03b09ac3b8982ff0bd01037d34dd
121 lines
4.0 KiB
YAML
121 lines
4.0 KiB
YAML
- hosts: localhost
|
|
tasks:
|
|
- name: Add bridge.o.o to inventory for playbook
|
|
add_host:
|
|
name: bridge.openstack.org
|
|
ansible_python_interpreter: python3
|
|
ansible_user: zuul
|
|
# Without setting ansible_host directly, mirror-workspace-git-repos
|
|
# gets sad because if delegate_to localhost and with add_host that
|
|
# ends up with ansible_host being localhost.
|
|
ansible_host: bridge.openstack.org
|
|
ansible_port: 22
|
|
|
|
- hosts: bridge.openstack.org
|
|
tasks:
|
|
- name: Encrypt log
|
|
when: infra_prod_playbook_encrypt_log|default(False)
|
|
block:
|
|
|
|
- name: Create temporary staging area for encrypted logs
|
|
tempfile:
|
|
state: directory
|
|
register: _encrypt_tempdir
|
|
|
|
- name: Copy log to tempdir as Zuul user
|
|
copy:
|
|
src: '/var/log/ansible/{{ playbook_name }}.log'
|
|
dest: '{{ _encrypt_tempdir.path }}'
|
|
owner: zuul
|
|
group: zuul
|
|
mode: '0644'
|
|
remote_src: yes
|
|
become: yes
|
|
|
|
- name: Encrypt logs
|
|
include_role:
|
|
name: encrypt-logs
|
|
vars:
|
|
encrypt_logs_files:
|
|
- '{{ _encrypt_tempdir.path }}/{{ playbook_name }}.log'
|
|
# Artifact URL should just point to root directory, so blank
|
|
encrypt_logs_artifact_path: ''
|
|
encrypt_logs_download_script_path: '{{ _encrypt_tempdir.path }}'
|
|
|
|
- name: Return logs
|
|
synchronize:
|
|
src: '{{ item[0] }}'
|
|
dest: '{{ item[1] }}'
|
|
mode: pull
|
|
verify_host: true
|
|
loop:
|
|
- ['{{ _encrypt_tempdir.path }}/{{ playbook_name }}.log.gpg', '{{ zuul.executor.log_root }}/{{ playbook_name }}.log.gpg']
|
|
- ['{{ _encrypt_tempdir.path }}/download-logs.sh' , '{{ zuul.executor.log_root }}/download-gpg-logs.sh']
|
|
|
|
always:
|
|
|
|
- name: Remove temporary staging
|
|
file:
|
|
path: '{{ _encrypt_tempdir.path }}'
|
|
state: absent
|
|
when: _encrypt_tempdir is defined
|
|
|
|
# Not using normal zuul job roles as bridge.openstack.org is not a
|
|
# test node with all the normal bits in place.
|
|
- name: Collect log output
|
|
synchronize:
|
|
dest: "{{ zuul.executor.log_root }}/{{ playbook_name }}.log"
|
|
mode: pull
|
|
src: "/var/log/ansible/{{ playbook_name }}.log"
|
|
verify_host: true
|
|
when: infra_prod_playbook_collect_log
|
|
|
|
- name: Return playbook log artifact to Zuul
|
|
when: infra_prod_playbook_collect_log
|
|
zuul_return:
|
|
data:
|
|
zuul:
|
|
artifacts:
|
|
- name: "Playbook Log"
|
|
url: "{{ playbook_name }}.log"
|
|
metadata:
|
|
type: text
|
|
|
|
# Save files locally on bridge
|
|
- name: Get original timestamp from file header
|
|
shell: |
|
|
head -1 /var/log/ansible/{{ playbook_name.log }} | sed -n 's/^Running \(.*\):.*$/\1/p'
|
|
args:
|
|
executable: /bin/bash
|
|
register: _log_timestamp
|
|
|
|
- name: Turn timestamp into a string
|
|
set_fact:
|
|
_log_timestamp: '{{ _log_timestamp.stdout | trim }}'
|
|
|
|
- name: Rename playbook log on bridge
|
|
when: not infra_prod_playbook_collect_log
|
|
become: yes
|
|
copy:
|
|
remote_src: yes
|
|
src: "/var/log/ansible/{{ playbook_name }}.log"
|
|
dest: "/var/log/ansible/{{ playbook_name }}.log.{{ _log_timestamp }}"
|
|
|
|
# Reset the access/modification time to the timestamp in the filename; this
|
|
# makes lining things up more logical
|
|
- name: Reset file time
|
|
file:
|
|
src: '/var/log/ansible/{{ playbook_name }}.log.{{ _log_timestamp }}'
|
|
state: touch
|
|
modification_time: '{{ _log_timestamp }}'
|
|
modification_time_format: '%Y-%m%-%dT%H:%M:%S'
|
|
access_time: '{{ _log_timestamp }}'
|
|
access_time_format: '%Y-%m%-%dT%H:%M:%S'
|
|
|
|
- name: Cleanup old playbook logs on bridge
|
|
when: not infra_prod_playbook_collect_log
|
|
become: yes
|
|
shell: |
|
|
find /var/log/ansible -name '{{ playbook_name }}.log.*' -type f -mtime +30 -delete
|
|
|