production-playbook logs : move to post-run step

If the production playbook times out, we don't get any logs collected
with the run.  By moving the log collection into a post-run step, we
should always get something copied to help us diagnose what is going
wrong.

Change-Id: I3e99b80e442db0cc87f8e8c9728b7697a5e4d1d3

playbooks/zuul/run-production-playbook-post.yaml

@@ -0,0 +1,86 @@
+- hosts: bridge.openstack.org
+  tasks:
+    - name: Encrypt log
+      when: infra_prod_playbook_encrypt_log|default(False)
+      block:
+
+        - name: Create temporary staging area for encrypted logs
+          tempfile:
+            state: directory
+          register: _encrypt_tempdir
+
+        - name: Copy log to tempdir as Zuul user
+          copy:
+            src: '/var/log/ansible/{{ playbook_name }}.log'
+            dest: '{{ _encrypt_tempdir.path }}'
+            owner: zuul
+            group: zuul
+            mode: '0644'
+            remote_src: yes
+          become: yes
+
+        - name: Encrypt logs
+          include_role:
+            name: encrypt-logs
+          vars:
+            encrypt_logs_files:
+              - '{{ _encrypt_tempdir.path }}/{{ playbook_name }}.log'
+            # Artifact URL should just point to root directory, so blank
+            encrypt_logs_artifact_path: ''
+            encrypt_logs_download_script_path: '{{ _encrypt_tempdir.path }}'
+
+        - name: Return logs
+          synchronize:
+            src: '{{ item[0] }}'
+            dest: '{{ item[1] }}'
+            mode: pull
+            verify_host: true
+          loop:
+            - ['{{ _encrypt_tempdir.path }}/{{ playbook_name }}.log.gpg', '{{ zuul.executor.log_root }}/{{ playbook_name }}.log.gpg']
+            - ['{{ _encrypt_tempdir.path }}/download-logs.sh' , '{{ zuul.executor.log_root }}/download-gpg-logs.sh']
+
+      always:
+
+        - name: Remove temporary staging
+          file:
+            path: '{{ _encrypt_tempdir.path }}'
+            state: absent
+          when: _encrypt_tempdir is defined
+
+    # Not using normal zuul job roles as bridge.openstack.org is not a
+    # test node with all the normal bits in place.
+    - name: Collect log output
+      synchronize:
+        dest: "{{ zuul.executor.log_root }}/{{ playbook_name }}.log"
+        mode: pull
+        src: "/var/log/ansible/{{ playbook_name }}.log"
+        verify_host: true
+      when: infra_prod_playbook_collect_log
+
+    - name: Return playbook log artifact to Zuul
+      when: infra_prod_playbook_collect_log
+      zuul_return:
+        data:
+          zuul:
+            artifacts:
+              - name: "Playbook Log"
+                url: "{{ playbook_name }}.log"
+                metadata:
+                  type: text
+
+    # If we aren't publishing logs through zuul then keep a set on
+    # bridge directly.
+    - name: Rename playbook log on bridge
+      when: not infra_prod_playbook_collect_log
+      become: yes
+      copy:
+        remote_src: yes
+        src: "/var/log/ansible/{{ playbook_name }}.log"
+        dest: "/var/log/ansible/{{ playbook_name }}.log.{{ _log_timestamp }}"
+
+    - name: Cleanup old playbook logs on bridge
+      when: not infra_prod_playbook_collect_log
+      become: yes
+      shell: |
+        find /var/log/ansible -name '{{ playbook_name }}.log.*' -type f -mtime +30 -delete
+

playbooks/zuul/run-production-playbook.yaml

@@ -15,7 +15,6 @@
   tasks:
     - name: Run the production playbook and capture logs
       block:
-
       - name: Get a current timestamp
         set_fact:
           _log_timestamp: "{{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%S') }}"
@@ -30,7 +29,6 @@
         register: _run
 
       always:
-
         - name: Send run stats
           shell: |
               # delta is in string format h:m:s.sss; convert to ms for statsd
@@ -40,88 +38,3 @@
               echo 'bridge.ansible.{{ zuul.job }}.rc:{{ _run.rc }}|g' | nc -w 1 -u graphite.opendev.org 8125
           args:
             executable: '/bin/bash'
-
-        - name: Encrypt log
-          when: infra_prod_playbook_encrypt_log|default(False)
-          block:
-
-            - name: Create temporary staging area for encrypted logs
-              tempfile:
-                state: directory
-              register: _encrypt_tempdir
-
-            - name: Copy log to tempdir as Zuul user
-              copy:
-                src: '/var/log/ansible/{{ playbook_name }}.log'
-                dest: '{{ _encrypt_tempdir.path }}'
-                owner: zuul
-                group: zuul
-                mode: '0644'
-                remote_src: yes
-              become: yes
-
-            - name: Encrypt logs
-              include_role:
-                name: encrypt-logs
-              vars:
-                encrypt_logs_files:
-                  - '{{ _encrypt_tempdir.path }}/{{ playbook_name }}.log'
-                # Artifact URL should just point to root directory, so blank
-                encrypt_logs_artifact_path: ''
-                encrypt_logs_download_script_path: '{{ _encrypt_tempdir.path }}'
-
-            - name: Return logs
-              synchronize:
-                src: '{{ item[0] }}'
-                dest: '{{ item[1] }}'
-                mode: pull
-                verify_host: true
-              loop:
-                - ['{{ _encrypt_tempdir.path }}/{{ playbook_name }}.log.gpg', '{{ zuul.executor.log_root }}/{{ playbook_name }}.log.gpg']
-                - ['{{ _encrypt_tempdir.path }}/download-logs.sh' , '{{ zuul.executor.log_root }}/download-gpg-logs.sh']
-
-          always:
-
-            - name: Remove temporary staging
-              file:
-                path: '{{ _encrypt_tempdir.path }}'
-                state: absent
-              when: _encrypt_tempdir is defined
-
-        # Not using normal zuul job roles as bridge.openstack.org is not a
-        # test node with all the normal bits in place.
-        - name: Collect log output
-          synchronize:
-            dest: "{{ zuul.executor.log_root }}/{{ playbook_name }}.log"
-            mode: pull
-            src: "/var/log/ansible/{{ playbook_name }}.log"
-            verify_host: true
-          when: infra_prod_playbook_collect_log
-
-        - name: Return playbook log artifact to Zuul
-          when: infra_prod_playbook_collect_log
-          zuul_return:
-            data:
-              zuul:
-                artifacts:
-                  - name: "Playbook Log"
-                    url: "{{ playbook_name }}.log"
-                    metadata:
-                      type: text
-
-        # If we aren't publishing logs through zuul then keep a set on
-        # bridge directly.
-        - name: Rename playbook log on bridge
-          when: not infra_prod_playbook_collect_log
-          become: yes
-          copy:
-            remote_src: yes
-            src: "/var/log/ansible/{{ playbook_name }}.log"
-            dest: "/var/log/ansible/{{ playbook_name }}.log.{{ _log_timestamp }}"
-
-        - name: Cleanup old playbook logs on bridge
-          when: not infra_prod_playbook_collect_log
-          become: yes
-          shell: |
-            find /var/log/ansible -name '{{ playbook_name }}.log.*' -type f -mtime +30 -delete
-

zuul.d/infra-prod.yaml

@@ -21,6 +21,7 @@
     abstract: true
     semaphores: infra-prod-playbook
     run: playbooks/zuul/run-production-playbook.yaml
+    post-run: playbooks/zuul/run-production-playbook-post.yaml
     required-projects:
       - opendev/system-config
     vars: