system-config/playbooks/zuul/run-production-playbook.yaml

- hosts: localhost
  tasks:
    - name: Add bridge.o.o to inventory for playbook
      add_host:
        name: bridge.openstack.org
        ansible_python_interpreter: python3
        ansible_user: zuul
        # Without setting ansible_host directly, mirror-workspace-git-repos
        # gets sad because if delegate_to localhost and with add_host that
        # ends up with ansible_host being localhost.
        ansible_host: bridge.openstack.org
        ansible_port: 22

- hosts: bridge.openstack.org
  tasks:
    - name: Run the production playbook and capture logs
      block:

      - name: Log a playbook start header
        become: yes
        shell: 'echo "Running {{ ansible_date_time.iso8601 }}: ansible-playbook -v -f {{ infra_prod_ansible_forks }} /home/zuul/src/opendev.org/opendev/system-config/playbooks/{{ playbook_name }}" > /var/log/ansible/{{ playbook_name }}.log'

      - name: Run specified playbook on bridge.o.o and redirect output
        become: yes
        shell: 'ansible-playbook -v -f {{ infra_prod_ansible_forks }} /home/zuul/src/opendev.org/opendev/system-config/playbooks/{{ playbook_name }} >> /var/log/ansible/{{ playbook_name }}.log'
        register: _run

      always:

        - name: Send run stats
          shell: |
              # delta is in string format h:m:s.sss; convert to ms for statsd
              {% set delta = _run.delta.split(':') %}
              {% set delta_ms = ((delta[0]|int * 60 * 60 * 1000) + (delta[1]|int * 60 * 1000) + (delta[2]|float * 1000)) | int %}
              echo 'bridge.ansible.{{ zuul.job }}.runtime:{{ delta_ms }}|ms' | nc -w 1 -u graphite.opendev.org 8125
              echo 'bridge.ansible.{{ zuul.job }}.rc:{{ _run.rc }}|g' | nc -w 1 -u graphite.opendev.org 8125              
          args:
            executable: '/bin/bash'

        - name: Encrypt log
          when: infra_prod_playbook_encrypt_log|default(False)
          block:

            - name: Create temporary staging area for encrypted logs
              tempfile:
                state: directory
              register: _encrypt_tempdir

            - name: Copy log to tempdir as Zuul user
              copy:
                src: '/var/log/ansible/{{ playbook_name }}.log'
                dest: '{{ _encrypt_tempdir.path }}'
                owner: zuul
                group: zuul
                mode: '0644'
                remote_src: yes
              become: yes

            - name: Encrypt logs
              include_role:
                name: encrypt-logs
              vars:
                encrypt_logs_files:
                  - '{{ _encrypt_tempdir.path }}/{{ playbook_name }}.log'
                # Artifact URL should just point to root directory, so blank
                encrypt_logs_artifact_path: ''
                encrypt_logs_download_script_path: '{{ _encrypt_tempdir.path }}'

            - name: Return logs
              synchronize:
                src: '{{ item[0] }}'
                dest: '{{ item[1] }}'
                mode: pull
                verify_host: true
              loop:
                - ['{{ _encrypt_tempdir.path }}/{{ playbook_name }}.log.gpg', '{{ zuul.executor.log_root }}/{{ playbook_name }}.log.gpg']
                - ['{{ _encrypt_tempdir.path }}/download-logs.sh' , '{{ zuul.executor.log_root }}/download-gpg-logs.sh']

          always:

            - name: Remove temporary staging
              file:
                path: '{{ _encrypt_tempdir.path }}'
                state: absent
              when: _encrypt_tempdir is defined

        # Not using normal zuul job roles as bridge.openstack.org is not a
        # test node with all the normal bits in place.
        - name: Collect log output
          synchronize:
            dest: "{{ zuul.executor.log_root }}/{{ playbook_name }}.log"
            mode: pull
            src: "/var/log/ansible/{{ playbook_name }}.log"
            verify_host: true
          when: infra_prod_playbook_collect_log

        - name: Return playbook log artifact to Zuul
          when: infra_prod_playbook_collect_log
          zuul_return:
            data:
              zuul:
                artifacts:
                  - name: "Playbook Log"
                    url: "{{ playbook_name }}.log"
                    metadata:
                      type: text

        # If we aren't publishing logs through zuul then keep a set on
        # bridge directly.
        - name: Rename playbook log on bridge
          when: not infra_prod_playbook_collect_log
          become: yes
          copy:
            remote_src: yes
            src: "/var/log/ansible/{{ playbook_name }}.log"
            dest: "/var/log/ansible/{{ playbook_name }}.log.{{ ansible_date_time.iso8601 }}"

        - name: Cleanup old playbook logs on bridge
          when: not infra_prod_playbook_collect_log
          become: yes
          shell: |
            find /var/log/ansible -name '{{ playbook_name }}.log.*' -type f -mtime +30 -delete