From ccf00b7673db04a01c6f8591db6c04e8f060b26f Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Fri, 11 Feb 2022 12:06:13 +1100 Subject: [PATCH] Base work for exporting encrypted logs Our production jobs currently only put their logging locally on the bastion host. This means that to help maintain a production system, you effectively need full access to the bastion host to debug any misbehaviour. We've long discussed publishing these Ansible runs as public logs, or via a reporting system (ARA, etc.) but, despite our best efforts at no_log and similar, we are not 100% sure that secret values may not leak. This is the infrastructure for an in-between solution, where we publish the production run logs encrypted to specific GPG public keys. Here we are capturing and encrypting the logs of the system-config-run-* jobs, and providing a small download script to automatically grab and unencrypt the log files. Obviously this is just to exercise the encryption/log-download path for these jobs, as the logs are public. Once this has landed, I will propose similar for the production jobs (because these are post-pipeline this takes a bit more fiddling and doens't run in CI). The variables will be setup in such a way that if someone wishes to help maintain a production system, they can add their public-key and then add themselves to the particular infra-prod-* job they wish to view the logs for. It is planned that the extant operators will be in the default list; however this is still useful over the status quo -- instead of having to search through the log history on the bastion host when debugging a failed run, they can simply view the logs from the failing build in Zuul directly. Depends-On: https://review.opendev.org/c/zuul/zuul-jobs/+/828818/ Change-Id: I5b9f9dd53eb896bb542652e8175c570877842584 --- playbooks/service-bridge.yaml | 7 ++ .../roles/encrypt-logs/defaults/main.yaml | 85 ++++++++++++++++++ .../zuul/roles/encrypt-logs/tasks/main.yaml | 27 ++++++ .../templates/download-logs.sh.j2 | 89 +++++++++++++++++++ playbooks/zuul/run-base.yaml | 27 ++++-- zuul.d/system-config-run.yaml | 1 + 6 files changed, 231 insertions(+), 5 deletions(-) create mode 100644 playbooks/zuul/roles/encrypt-logs/defaults/main.yaml create mode 100644 playbooks/zuul/roles/encrypt-logs/tasks/main.yaml create mode 100644 playbooks/zuul/roles/encrypt-logs/templates/download-logs.sh.j2 diff --git a/playbooks/service-bridge.yaml b/playbooks/service-bridge.yaml index 72e0c694b1..164d924773 100644 --- a/playbooks/service-bridge.yaml +++ b/playbooks/service-bridge.yaml @@ -32,3 +32,10 @@ - name: Install rackspace DNS backup tool include_role: name: rax-dns-backup + + - name: Make ansible log directory + file: + path: '/var/log/ansible' + state: directory + owner: root + mode: 0755 diff --git a/playbooks/zuul/roles/encrypt-logs/defaults/main.yaml b/playbooks/zuul/roles/encrypt-logs/defaults/main.yaml new file mode 100644 index 0000000000..445165e3f1 --- /dev/null +++ b/playbooks/zuul/roles/encrypt-logs/defaults/main.yaml @@ -0,0 +1,85 @@ +# Anyone who wants to be able to read encrypted published logs should +# have an entry in this variable in the format +# +# - name: +# key_id: +# gpg_asc: +# +encrypt_logs_keys: + - name: 'ianw' + key_id: '0x9615aec8' + gpg_asc: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + + mQINBFQO6lIBEADINS0UyBzR0ljAfYy6h4eh266Hi4R5Q6hw1aZPezz+yrFu+1po + XWlHBBGs2BRKakM9vY6JkedX13iYJo0D0TACIQFqICXI91wLLoT4RMwTK5XWhJoq + OnMqA578fepgKWivd06lTmd3KwVy6stT/E4i1KWpntHymwiWQ25SOOccGA4cy7qf + kGdNDupACXFp5Sa61fBgTLlf9JDR8t4hOqEpb3czDOjSS+LGBdBYzA3DyxBySJk5 + yhNPug2UNnWv0i+mOuehp1no/VNJzCAvCDjSGLD5fVRBpPCPBGhvr5oJ1VvLU51B + 7kuV+pHIurn5dMtocGhgenuwHUouAS+IiYaywlaJ/N9s5GT8jnYGht+DcxyuHFj2 + ZaTfBwDa3DrSTisKBMwfRMf9zKkwDRERM0C0ReknS1ACwuIlZhDKjlHx1Rv67uuz + DIQieX+kThF5YPqFNm52bs4fHfcuKDBU2aCpmmASR8tr15olC8LhW/PlExix8rY7 + ALUdySejLEdY86xnrqV9g2VSHwdyXDOKMxbwQO5l4IsQgZfdcJJ6KK8sAVvO3Ftc + MZzt0oebF3ocO6Azm8de2sx3bVNeod3K1fERG83u7OuAuJmWoHMbnhRqmt/c3Pua + ZC4Z+c1mBYe9ZpL/zfUooWoS6H9F9J0tkvoSYXsxSkv3BdGluqiGO0kuXwARAQAB + tB1JYW4gV2llbmFuZCA8aWFuQHdpZW5hbmQub3JnPokCOQQTAQgAJAIbAwULCQgH + AwUVCgkICwUWAgMBAAIeAQIXgAUCVA7sxQIZAQAKCRC2pvuLlhWuyOQ9D/iYeO7M + XojUPYTnAcc7fj2ltXKDhcKmslhB7gqg1y3kLGK8Veys407WAOg124rI3dSViUQ3 + uEPDBYNc1YSfxIRLPB5tlNYYlAOmuSxyMn7pHBSM/2ZJwAws4H7HrdsDoEJ59/FQ + B/T3fhKTAYqYU3OHJNrCTAizLaD4DHQsLB3ZngL2HArRYJS05Kt3f4VnlIPIkCO5 + lxbLdUvLzgW5fUIRG6zkHJ7Ws9W4b7Lrfmu/HoqNkZVbY6c2uYziiPvi0xX5eVa/ + EFHerMJVg41Kh3xIAkWmphSMiKHcrAxgJNsiV3LnrbemW4vqsh8HjbdChBkLdd/l + jiIsr46oILyry32L5TN1PIVZF5cgsEOmaQXOktT6Ot062j5CJ/uktW2ZThvu75O1 + q3e6ai5lU9hSLUlyXs0GQUyGz+iW7rQRYkFA0ei0/Kj27h+SJHCvXQlBgDMoYWzz + 5P76YN+3GXILA55L56+e7+oowv7OnPJYiNfxsqSmP5DfuI8aZV3e6zowKdAJcQKX + modQC0D6HiIAJwfsSbHhCb4S/PmpKxneoKYBophnxTQoUsCgN3ohTj48P8ghHZso + GwKxYwj5c8T2MYKtYp8jZFj7e22JrezK6mxNSGRR3kfSxQkSFbMOzhQEc+yBCE5g + A1k69gpGnkhNt4LOVyvCw+23SsT1HK04UrWkiEYEEBEIAAYFAlQO7bcACgkQWDlS + U/gp6edmbQCdEhGjPxS9ThH956Qu4nXhaEeNBJgAoKbnfznTgEoU0KDJwfjyPeo3 + p0L5tB1JYW4gV2llbmFuZCA8aWFud0BkZWJpYW4ub3JnPokCNwQTAQgAIQIbAwUL + CQgHAwUVCgkICwUWAgMBAAIeAQIXgAUCVA7sxAAKCRC2pvuLlhWuyGUBEACT8QUp + OeF7zuZyEb9/g0zBkinv2SXH3DlcGwTnD9XLq6NfmETIM8zAz+WCeFtE2U9cMdP6 + JAQfhBr49WFYf1fIw6fZvlALSekRLMLlHA6E7P+bERITxahkhsf1cGmNMewgY8SD + 7/YDSh4hzaou/2XEPsjOa49kwkylrNmA7+57UbYXINtlc2mlO3vl8ZFaNfRiBnVO + fkGGYxTs2h99Apqixw41zx0Jk+ebB+xUdP83aswEwP5CpfADDtJiRORhBUeoqBkv + y9UKpwMd6Mtn7+mDDDMI3/NjODsxqxyDDRcHhUhu9mXZJLT1GiaTL2bS5NAn6g07 + 7n6mvxN58YLizy+TwK1z30Ls1+3WEppjq7RGbXL2fn7qHnddlp0lZOE2P6VcGCJG + dfH0tdbaYKWMT7dMBgKRGZIxBw3fToqrTG5fjkmON31A+mloU/JusuYSeOk9H/MZ + r8HffeGC/R4ZUpBiNTEvfEkdZGW+WOGFYjjYSBmKw98UpahPW3cmDKaqaDizY6VQ + MxcN66eH40arvVID+YQHCkiABgfeHWhP92bIAIIPphmibXg8XMR0gUCTtZjy4usQ + 7xqoSvCt/K5Om9OFoSF/CYHB/31r7RzuKQqED+DcIp9QmnLFqeKFcLFEuhHv0llt + Gd1m/UQy7NtzfRYZXQmjOpSgrGvwqYJLIe3ProhGBBARCAAGBQJUDu23AAoJEFg5 + UlP4Kenn748AoLtqkDtDcf9pwupm+PCiDD6xgTb1AKC6zwJFCY+Y5JsBFKp/VUz+ + 2bA7I7kCDQRUDupSARAAyJbRcgcnwFNvutHiNCnaOgqE6zSV6xTebcPJEqIBlnGP + Svk6XLWgbZ6IKbuXIHjn+RggtzDuKnpj+c5QFZpwNuTsJ43t0erIm8FmuvHkUaQl + gD/7i4YZulUckN7E41i8BoqD3IcpeEjI/2VrPjhVvzWU+MxRpVrF5VdCWa/tSqFl + 8Xr+/nNR7VW0ojr/eplovXaxBXOMf2vraynwny4ftvW7oaJ6KlXuP3BSqpePAd72 + kz33s9k3X/TeUruTtIn/dWXyT36b67IaZRdzxBHguYcKCy9+r+f1IIssq6kjgZi0 + o2VijZ2DZvR0YB0cSe1FHKCmaqxx9I+wajBIw/bgFf3gDmABPpTK0Z4T3oIrUGoF + q7JXcqT9X+cZidimZSdE6xgiMzS2xSNXf1bmK/6BslU3/PiwMy7MDUM02z1pGKVK + xV+5Al7Tf/gFYMrpQ6r6rDAVj9Ube+ORTIfJcWKbBJ1RLIo8NA6/K+QqezMsyX9n + IZXM1fdQVm9OsGgzdT8h2kMLzm1JAj7yahAnig33MKp/IUfcsjLsR8GnkMTxv8Tq + O8X5AHyxk+sbzSLmFpT6nGGsGL4o+3rfPyQmhWkkuxjBRQeUQABCLgF01tfzlXch + JwLygD/ENAeYMDrNaTp9sgWsX7QZvHE/cLLk8bKqtziGRBtsTgBAjIHnKFSJAMEA + EQEAAYkCHwQYAQgACQUCVA7qUgIbDAAKCRC2pvuLlhWuyKqRD/4zm/PsFH3njOsJ + MrOwidxvlKNG+x6GZc6W6yvO1OyVZVtbsSwVAJfFQZejmj0NSL1sl/isQlQxBjXZ + TFjv3sQhUt+J6tOI7SzneP0iFXvhyuz3ffe0myG1pBKCgISmUDPEADYd9D7puyv6 + sZe56wiMXxHOiTrlHLHGFpmezfgybX2f/sQpZcUP4titLhVCt6A2cliHQcCoQjOI + FibNfks3jgql6YtF/JPkThxOcuT5BF59hVoSh41iLlO5feMJYIa8IQk3o7dzSVCX + EKotxotuYQ/bvmwZDT50TIJY3GzrpQsLxcM5+jaRB+S4rF91tAQ6msvdaSbhoARH + VEBMZAFM4257XMZXghgqdbHgtU8IEpL7rGwuIyoC/pEsLkbnzKuwthWag1U6KmoO + wevlH3u9Lj57zJjsn2RTrHHzVNIlIe8tJuInnacAoYq1lsmj5NCrZ8wpTSnv7C+o + rPQDmnnQz9MoUruyutd51GCMUTv4KthSYElgS9y18BPT0F5cHWkNBBT+W3NEPjgQ + 1SAxJ3dpqZgDlmWBO9XJ5rhb5rUiUDc03Dmnq7qLZtHQEXTipiUkuyoF24hisJ7+ + XgLPdWwyuOjfSc6foik6xYSuR1duxypmb9BidTmVPtQtG9uRvpc6vc7nRoUJKN8U + YyJrcbCb0lGKJnpdWIXnDldS91E0Lw== + =jbiD + -----END PGP PUBLIC KEY BLOCK----- + +# This is the default list of keys from ``encrypt_log_keys`` that wish +# to always be a recipient of encrypted logs. Others can add +# themselves to particular prod jobs of interest individually. +encrypt_logs_recipients: + - ianw + diff --git a/playbooks/zuul/roles/encrypt-logs/tasks/main.yaml b/playbooks/zuul/roles/encrypt-logs/tasks/main.yaml new file mode 100644 index 0000000000..ba49cf836b --- /dev/null +++ b/playbooks/zuul/roles/encrypt-logs/tasks/main.yaml @@ -0,0 +1,27 @@ +- name: Encrypt file + include_role: + name: encrypt-file + vars: + encrypt_file: '{{ encrypt_logs_files }}' + encrypt_file_keys: '{{ encrypt_logs_keys }}' + encrypt_file_recipients: '{{ encrypt_logs_recipients + encrypt_logs_job_recipients|default([]) }}' + +- name: Write download script + template: + src: download-logs.sh.j2 + dest: '{{ encrypt_logs_download_script_path }}/download-logs.sh' + mode: 0755 + vars: + encrypt_logs_download_api: 'https://zuul.opendev.org/api/tenant/{{ zuul.tenant }}' + +- name: Return artifact + zuul_return: + data: + zuul: + artifacts: + # This is parsed by the log download script above, so any + # changes to format must be accounted for there too. + - name: Encrypted logs + url: '{{ encrypt_logs_artifact_path }}' + metadata: + logfiles: "{{ encrypt_logs_files | map('basename') | map('regex_replace', '^(.*)$', '\\1.gpg') | list }}" diff --git a/playbooks/zuul/roles/encrypt-logs/templates/download-logs.sh.j2 b/playbooks/zuul/roles/encrypt-logs/templates/download-logs.sh.j2 new file mode 100644 index 0000000000..69816933e0 --- /dev/null +++ b/playbooks/zuul/roles/encrypt-logs/templates/download-logs.sh.j2 @@ -0,0 +1,89 @@ +#!/bin/bash + +set -e + +ZUUL_API=${ZUUL_API:-"{{ encrypt_logs_download_api }}"} +ZUUL_BUILD_UUID=${ZUUL_BUILD_UUID:-"{{ zuul.build }}"} +{% raw %} +ZUUL_API_URL=${ZUUL_API}/build/${ZUUL_BUILD_UUID} + +(( ${BASH_VERSION%%.*} >= 4 )) || { echo >&2 "bash >=4 required to download."; exit 1; } +command -v python3 >/dev/null 2>&1 || { echo >&2 "Python3 is required to download."; exit 1; } +command -v curl >/dev/null 2>&1 || { echo >&2 "curl is required to download."; exit 1; } + +function log { + echo "$(date -Iseconds) | $@" +} + +function get_urls { + /usr/bin/env python3 - < /dev/null + +log "Getting logs from ${ZUUL_BUILD_ID}" +for (( i=1; i<$len; i++ )); do + file="${files[i]}" + printf -v _out " %-80s [ %04d/%04d ]" "${file}" "${i}" $(( len -1 )) + log "$_out" + save_file $file +done + +for f in ${DOWNLOAD_DIR}/*.gpg; do + log "Decrypting $(basename $f)" + gpg --output ${f/.gpg/} --decrypt ${f} + rm ${f} +done + +popd >/dev/null + +log "Download to ${DOWNLOAD_DIR} complete!" +{% endraw %} diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index 66a1495baa..d6d7c61a46 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -92,18 +92,35 @@ - name: Display group membership command: ansible localhost -m debug -a 'var=groups' - name: Run base.yaml - command: ansible-playbook -f 50 -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/base.yaml + shell: 'ansible-playbook -f 50 -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/base.yaml 2>&1 | tee /var/log/ansible/base.yaml.log' - name: Run bridge service playbook - command: ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/service-bridge.yaml + shell: 'ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/service-bridge.yaml 2>&1 | tee /var/log/ansible/service-bridge.yaml.log' - name: Run dstat logger playbook - command: ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/service-dstatlogger.yaml + shell: 'ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/service-dstatlogger.yaml 2>&1 | tee /var/log/ansible/service-dstatlogger.yaml.log' + - name: Run playbook when: run_playbooks is defined loop: "{{ run_playbooks }}" - command: "ansible-playbook -f 50 -v /home/zuul/src/opendev.org/opendev/system-config/{{ item }}" + shell: "ansible-playbook -f 50 -v /home/zuul/src/opendev.org/opendev/system-config/{{ item }} 2>&1 | tee /var/log/ansible/{{ item | basename }}.log" + + - name: Build list of playbook logs + find: + paths: '/var/log/ansible' + patterns: '*.yaml.log' + register: _run_playbooks_logs + + - name: Encrypt playbook logs + when: run_playbooks is defined + include_role: + name: encrypt-logs + vars: + encrypt_logs_files: '{{ _run_playbooks_logs.files | map(attribute="path") | list }}' + encrypt_logs_artifact_path: 'bridge.openstack.org/ansible' + encrypt_logs_download_script_path: '/var/log/ansible' + - name: Run test playbook when: run_test_playbook is defined - shell: "ANSIBLE_ROLES_PATH=/home/zuul/src/opendev.org/opendev/system-config/playbooks/roles ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/{{ run_test_playbook }}" + shell: "ANSIBLE_ROLES_PATH=/home/zuul/src/opendev.org/opendev/system-config/playbooks/roles ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/{{ run_test_playbook }} 2>&1 | tee /var/log/ansible/{{ run_test_playbook | basename }}.log" - name: Generate testinfra extra data fixture set_fact: diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 219996f1fc..0cde0543b5 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -28,6 +28,7 @@ '{{ zuul.project.src_dir }}/test-results.html': logs '{{ zuul.project.src_dir }}/inventory/base/gate-hosts.yaml': logs '/var/log/screenshots': logs + '/var/log/ansible': logs # Note: the following two jobs implement the variant-based multiple # inheritance trick. Both of these variants will always apply,