diff --git a/roles/aiap-gather-runner-vm-logs/defaults/main.yaml b/roles/aiap-gather-runner-vm-logs/defaults/main.yaml new file mode 100644 index 000000000..f5f7abb9b --- /dev/null +++ b/roles/aiap-gather-runner-vm-logs/defaults/main.yaml @@ -0,0 +1,13 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kubeconfig: "{{ airshipctl_config_dir_default | default(ansible_env.HOME) }}/.airship/kubeconfig" diff --git a/roles/aiap-gather-runner-vm-logs/tasks/main.yaml b/roles/aiap-gather-runner-vm-logs/tasks/main.yaml new file mode 100644 index 000000000..97f934d2f --- /dev/null +++ b/roles/aiap-gather-runner-vm-logs/tasks/main.yaml @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## This task will populate kctl_context_list variable with list of contexts if they exist + +- name: "Download logs from runner container" + include_tasks: save-runner-logs.yaml + +- name: "Downloads runner logs to executor" + synchronize: + src: "{{ logs_dir }}/aiap-runner-logs" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True + tags: [zuul] diff --git a/roles/aiap-gather-runner-vm-logs/tasks/save-runner-logs.yaml b/roles/aiap-gather-runner-vm-logs/tasks/save-runner-logs.yaml new file mode 100644 index 000000000..2d938e729 --- /dev/null +++ b/roles/aiap-gather-runner-vm-logs/tasks/save-runner-logs.yaml @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: "creating directory for pod logs" + file: + path: "{{ logs_dir }}/aiap-runner-logs" + state: directory + +- name: "retrieve all container logs, current and previous (if they exist)" + shell: |- + set -e + export KUBECONFIG="{{ kubeconfig }}" + kubectl cp airship-in-a-pod:/tmp/logs /tmp/logs/aiap-runner-logs -c runner + args: + executable: /bin/bash + ignore_errors: True + +... diff --git a/tools/airship-in-a-pod/runner/Dockerfile b/tools/airship-in-a-pod/runner/Dockerfile index 290d8da50..c3fb0a0ca 100644 --- a/tools/airship-in-a-pod/runner/Dockerfile +++ b/tools/airship-in-a-pod/runner/Dockerfile @@ -17,7 +17,14 @@ RUN apt-get update ;\ ca-certificates \ gnupg-agent \ libvirt-clients \ - gettext-base ;\ + gettext-base \ + wget \ + iptables \ + rsync \ + make; \ + pip3 install --upgrade ansible; \ + pip3 install --upgrade netaddr; \ + pip3 install --upgrade yq; \ curl -sSLo /usr/local/bin/kubectl "${kubectl_url}" ;\ chmod +x /usr/local/bin/kubectl ;\ rm -rf /var/lib/apt/lists/* diff --git a/tools/airship-in-a-pod/runner/assets/entrypoint.sh b/tools/airship-in-a-pod/runner/assets/entrypoint.sh index 3afb7469b..52d834149 100755 --- a/tools/airship-in-a-pod/runner/assets/entrypoint.sh +++ b/tools/airship-in-a-pod/runner/assets/entrypoint.sh @@ -17,6 +17,9 @@ set -ex /signal_status "runner" "RUNNING" success=false function reportStatus() { + # Run the get-logs script while inside the airshipctl repo to run the + # ansible log gathering playbooks before sending the signal status + /get-logs.sh if [[ "$success" == "false" ]]; then /signal_status "runner" "FAILED" else @@ -103,4 +106,5 @@ fi ./tools/deployment/25_deploy_gating.sh + success=true diff --git a/tools/airship-in-a-pod/runner/assets/get-logs.sh b/tools/airship-in-a-pod/runner/assets/get-logs.sh new file mode 100755 index 000000000..d758eb295 --- /dev/null +++ b/tools/airship-in-a-pod/runner/assets/get-logs.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -xe + +export AIRSHIPCTL_WS=${AIRSHIPCTL_WS:-$PWD} +export TMP_DIR=${TMP_DIR:-"$(dirname "$(mktemp -u)")"} + +ANSIBLE_CFG=${ANSIBLE_CFG:-"${HOME}/.ansible.cfg"} +ANSIBLE_HOSTS=${ANSIBLE_HOSTS:-"${TMP_DIR}/ansible_hosts"} +PLAYBOOK_CONFIG=${PLAYBOOK_CONFIG:-"${TMP_DIR}/config.yaml"} + +mkdir -p "$TMP_DIR" +envsubst <"${AIRSHIPCTL_WS}/tools/gate/config_template.yaml" > "$PLAYBOOK_CONFIG" + + +echo "primary ansible_host=localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3" > "$ANSIBLE_HOSTS" +printf "[defaults]\nroles_path = %s/roles:zuul-jobs/roles\n" "$AIRSHIPCTL_WS" > "$ANSIBLE_CFG" + +sudo -E ansible-playbook -i "$ANSIBLE_HOSTS" \ + playbooks/airship-collect-logs.yaml \ + -e @"$PLAYBOOK_CONFIG" \ + -e 'log_roles="[\"gather-system-logs\", \"airship-gather-libvirt-logs\", \"airship-gather-runtime-logs\", \"airship-airshipctl-gather-configs\", \"describe-kubernetes-objects\", \"airship-gather-pod-logs\"]"' \ No newline at end of file diff --git a/tools/airship-in-a-pod/scripts/11_build_images.sh b/tools/airship-in-a-pod/scripts/11_build_images.sh index 354092a25..2b2d2a6af 100755 --- a/tools/airship-in-a-pod/scripts/11_build_images.sh +++ b/tools/airship-in-a-pod/scripts/11_build_images.sh @@ -35,43 +35,48 @@ done # Now that we have built/pulled the images, lets change the imagePullPolicy to # Never to be 100% confident they are used -echo "- op: add +echo "- op: replace path: \"/spec/containers/0/imagePullPolicy\" value: Never -- op: add +- op: replace path: \"/spec/containers/1/imagePullPolicy\" value: Never -- op: add +- op: replace path: \"/spec/containers/2/imagePullPolicy\" value: Never -- op: add +- op: replace path: \"/spec/containers/3/imagePullPolicy\" value: Never -- op: add +- op: replace path: \"/spec/containers/4/imagePullPolicy\" value: Never -- op: add +- op: replace path: \"/spec/containers/5/imagePullPolicy\" value: Never -- op: add +- op: replace path: \"/spec/containers/6/imagePullPolicy\" value: Never -" >> examples/airshipctl/patchset.yaml +- op: replace + path: \"/spec/containers/7/imagePullPolicy\" + value: Never -# Also add the patchset to the environment variables +" >> examples/airshipctl/replacements.yaml + +# Also replace the patchset to the environment variables # while being sure to escape the slashes from the ref echo "- op: replace path: \"/spec/containers/4/env/6/value\" value: $AIRSHIPCTL_REF -" >> examples/airshipctl/patchset.yaml +" >> examples/airshipctl/replacements.yaml -popd || exit + +popd || exit \ No newline at end of file diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 64a5c4251..26012f2ca 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -187,6 +187,7 @@ - airship-airshipctl-gather-configs - describe-kubernetes-objects - airship-gather-pod-logs + - aiap-gather-runner-vm-logs voting: false - job: name: airship-airshipctl-gate-script-runner-docker