diff --git a/test-ansible-functional.sh b/test-ansible-functional.sh index f9be4435..8755945b 100755 --- a/test-ansible-functional.sh +++ b/test-ansible-functional.sh @@ -96,6 +96,14 @@ function execute_ansible_playbook { } +function gate_job_exit_tasks { + # This environment variable captures the exit code + # which was present when the trap was initiated. + # This would be the success/failure of the test. + export TEST_EXIT_CODE=$? + source "${COMMON_TESTS_PATH}/test-log-collect.sh" +} + ## Main ---------------------------------------------------------------------- # NOTE(mhayden): CentOS images in the gate have several slow mirrors enabled @@ -112,6 +120,9 @@ fi source "${COMMON_TESTS_PATH}/test-ansible-env-prep.sh" setup_ara +# Set gate job exit traps, this is run regardless of exit state when the job finishes. +trap gate_job_exit_tasks EXIT + # Prepare the extra CLI parameters used in each execution set_ansible_parameters diff --git a/test-log-collect.sh b/test-log-collect.sh index 6e1a71b7..b4e79de4 100755 --- a/test-log-collect.sh +++ b/test-log-collect.sh @@ -33,19 +33,21 @@ export TESTING_HOME=${TESTING_HOME:-$HOME} ## Main ---------------------------------------------------------------------- -echo "#### BEGIN LOG COLLECTION ###" +if [[ -d "/etc/nodepool" ]]; then -mkdir -vp \ + echo "#### BEGIN LOG COLLECTION ###" + + mkdir -vp \ "${WORKING_DIR}/logs/host" \ "${WORKING_DIR}/logs/openstack" \ "${WORKING_DIR}/logs/etc/host" \ "${WORKING_DIR}/logs/etc/openstack" \ -# NOTE(mhayden): We use sudo here to ensure that all logs are copied. -sudo ${RSYNC_CMD} /var/log/ "${WORKING_DIR}/logs/host" || true -if [ -d "/openstack/log" ]; then + # NOTE(mhayden): We use sudo here to ensure that all logs are copied. + sudo ${RSYNC_CMD} /var/log/ "${WORKING_DIR}/logs/host" || true + if [ -d "/openstack/log" ]; then sudo ${RSYNC_CMD} /openstack/log/ "${WORKING_DIR}/logs/openstack" || true -fi + fi # NOTE(cloudnull): This is collection thousands of files resulting in infra upload # issues. To remove immediate pressure this is being stopped and @@ -63,61 +65,63 @@ fi # done # fi -# NOTE(mhayden): All of the files must be world-readable so that the log -# pickup jobs will work properly. Without this, you get a "File not found" -# when trying to read the files in the job results. -# NOTE(odyssey4me): Using '--chown $(whoami) --chmod=ugo+rX' in the rsync -# CMD to achieve this would be optimal, but the CentOS version of rsync -# (3.0.x) does not support that option. -sudo chmod -R ugo+rX "${WORKING_DIR}/logs/" -sudo chown -R $(whoami) "${WORKING_DIR}/logs/" + # NOTE(mhayden): All of the files must be world-readable so that the log + # pickup jobs will work properly. Without this, you get a "File not found" + # when trying to read the files in the job results. + # NOTE(odyssey4me): Using '--chown $(whoami) --chmod=ugo+rX' in the rsync + # CMD to achieve this would be optimal, but the CentOS version of rsync + # (3.0.x) does not support that option. + sudo chmod -R ugo+rX "${WORKING_DIR}/logs/" + sudo chown -R $(whoami) "${WORKING_DIR}/logs/" -if [ ! -z "${ANSIBLE_LOG_DIR}" ]; then + if [ ! -z "${ANSIBLE_LOG_DIR}" ]; then mkdir -p "${WORKING_DIR}/logs/ansible" ${RSYNC_CMD} "${ANSIBLE_LOG_DIR}/" "${WORKING_DIR}/logs/ansible" || true -fi + fi -# Rename all files gathered to have a .txt suffix so that the compressed -# files are viewable via a web browser in OpenStack-CI. -find "${WORKING_DIR}/logs/" -type f ! -name '*.html' -exec mv {} {}.txt \; + # Rename all files gathered to have a .txt suffix so that the compressed + # files are viewable via a web browser in OpenStack-CI. + find "${WORKING_DIR}/logs/" -type f ! -name '*.html' -exec mv {} {}.txt \; -# Get the ara sqlite database -${RSYNC_CMD} "${TESTING_HOME}/.ara/ansible.sqlite" "${WORKING_DIR}/logs/" || true + # Get the ara sqlite database + ${RSYNC_CMD} "${TESTING_HOME}/.ara/ansible.sqlite" "${WORKING_DIR}/logs/" || true -# Generate the ARA report -# In order to reduce the quantity of unnecessary log content -# being kept in OpenStack-Infra we only generate the ARA report -# when the test result is a failure. The ARA sqlite database is -# still available for self generation if desired for successful -# tests. -if [[ "${TEST_EXIT_CODE}" != "0" ]]; then + # Generate the ARA report + # In order to reduce the quantity of unnecessary log content + # being kept in OpenStack-Infra we only generate the ARA report + # when the test result is a failure. The ARA sqlite database is + # still available for self generation if desired for successful + # tests. + if [[ "${TEST_EXIT_CODE}" != "0" ]]; then echo "Generating ARA report due to non-zero exit code (${TEST_EXIT_CODE})." ${ARA_CMD} "${WORKING_DIR}/logs/ara" || true -else + else echo "Not generating ARA report due to test pass." -fi + fi -# Get a dmesg output so we can look for kernel failures -dmesg > "${WORKING_DIR}/logs/dmesg.log.txt" || true + # Get a dmesg output so we can look for kernel failures + dmesg > "${WORKING_DIR}/logs/dmesg.log.txt" || true -# output ram usage -free -m > "${WORKING_DIR}/logs/memory-available.txt" || true + # output ram usage + free -m > "${WORKING_DIR}/logs/memory-available.txt" || true -# Redhat package debugging -if which yum &>/dev/null || which dnf &>/dev/null; then + # Redhat package debugging + if which yum &>/dev/null || which dnf &>/dev/null; then # Prefer dnf over yum for CentOS. which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum' sudo $RHT_PKG_MGR repolist -v > "${WORKING_DIR}/logs/redhat-rpm-repolist.txt" || true sudo $RHT_PKG_MGR list installed > "${WORKING_DIR}/logs/redhat-rpm-list-installed.txt" || true -# SUSE package debugging -elif which zypper &>/dev/null; then + # SUSE package debugging + elif which zypper &>/dev/null; then sudo zypper lr -d > "${WORKING_DIR}/logs/suse-zypper-repolist.txt" || true sudo zypper pa -i > "${WORKING_DIR}/logs/suse-zypper-list-installed.txt" || true + fi + + # Compress the files gathered so that they do not take up too much space. + # We use 'command' to ensure that we're not executing with some sort of alias. + command gzip --best --recursive "${WORKING_DIR}/logs/" + + echo "#### END LOG COLLECTION ###" + fi - -# Compress the files gathered so that they do not take up too much space. -# We use 'command' to ensure that we're not executing with some sort of alias. -command gzip --best --recursive "${WORKING_DIR}/logs/" - -echo "#### END LOG COLLECTION ###" diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index f2328440..215a6a85 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -112,7 +112,6 @@ - ^Vagrantfile timeout: 5400 # 90 mins run: zuul.d/playbooks/run - post-run: zuul.d/playbooks/post vars: tox_env: functional @@ -166,5 +165,4 @@ to execute the tests from must be specified in the ``osa_test_repo`` variable. run: zuul.d/playbooks/cross-repo-run - post-run: zuul.d/playbooks/cross-repo-post nodeset: ubuntu-xenial diff --git a/zuul.d/playbooks/cross-repo-post.yml b/zuul.d/playbooks/cross-repo-post.yml deleted file mode 100644 index 52305c7f..00000000 --- a/zuul.d/playbooks/cross-repo-post.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -# Copyright 2017, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- hosts: all - tasks: - - name: Run log collection script - shell: | - echo "TEST_EXIT_CODE: ${TEST_EXIT_CODE}" - if [[ -d "tests/common" ]]; then - source tests/common/test-log-collect.sh - else - echo "The common tests repo has not been cloned yet. Logs cannot be collected without it." - fi - args: - executable: "/bin/bash" - chdir: "{{ ansible_user_dir }}/src/git.openstack.org/{{ osa_test_repo }}" - environment: - TEST_EXIT_CODE: "{{ zuul_success }}" - - - name: Copy logs back to the executor - synchronize: - src: "{{ ansible_user_dir }}/src/git.openstack.org/{{ osa_test_repo }}/logs/" - dest: "{{ zuul.executor.log_root }}" - mode: pull diff --git a/zuul.d/playbooks/post.yml b/zuul.d/playbooks/post.yml deleted file mode 100644 index 27b9ddf0..00000000 --- a/zuul.d/playbooks/post.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -# Copyright 2017, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- hosts: all - tasks: - - name: Run log collection script - shell: | - echo "TEST_EXIT_CODE: ${TEST_EXIT_CODE}" - if [[ -d "tests/common" ]]; then - source tests/common/test-log-collect.sh - else - echo "The common tests repo has not been cloned yet. Logs cannot be collected without it." - fi - args: - executable: "/bin/bash" - chdir: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}" - environment: - TEST_EXIT_CODE: "{{ zuul_success }}" - - - name: Copy logs back to the executor - synchronize: - src: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}/logs/" - dest: "{{ zuul.executor.log_root }}" - mode: pull