diff --git a/scripts/gate-check-commit.sh b/scripts/gate-check-commit.sh index 125e1968bb..0ccbf9b2fc 100755 --- a/scripts/gate-check-commit.sh +++ b/scripts/gate-check-commit.sh @@ -74,9 +74,6 @@ info_block "Checking for required libraries." 2> /dev/null || source "${OSA_CLON ## Main ---------------------------------------------------------------------- -# Set gate job exit traps, this is run regardless of exit state when the job finishes. -trap gate_job_exit_tasks EXIT - # Log some data about the instance and the rest of the system log_instance_info diff --git a/scripts/scripts-library.sh b/scripts/scripts-library.sh index ff36bf9ffd..fe8225b467 100755 --- a/scripts/scripts-library.sh +++ b/scripts/scripts-library.sh @@ -22,13 +22,14 @@ ANSIBLE_PARAMETERS=${ANSIBLE_PARAMETERS:-""} STARTTIME="${STARTTIME:-$(date +%s)}" COMMAND_LOGS=${COMMAND_LOGS:-"/openstack/log/ansible_cmd_logs"} +ZUUL_PROJECT="${ZUUL_PROJECT:-}" GATE_EXIT_LOG_COPY="${GATE_EXIT_LOG_COPY:-false}" GATE_EXIT_LOG_GZIP="${GATE_EXIT_LOG_GZIP:-true}" GATE_EXIT_RUN_ARA="${GATE_EXIT_RUN_ARA:-true}" GATE_EXIT_RUN_DSTAT="${GATE_EXIT_RUN_DSTAT:-true}" # If this is a gate node from OpenStack-Infra Store all logs into the # execution directory after gate run. -if [[ -d "/etc/nodepool" ]]; then +if [[ -n "$ZUUL_PROJECT" ]]; then GATE_EXIT_LOG_COPY=true fi @@ -137,9 +138,9 @@ function gate_job_exit_tasks { fi GATE_LOG_DIR="${OSA_CLONE_DIR:-$(dirname $0)/..}/logs" mkdir -p "${GATE_LOG_DIR}/host" "${GATE_LOG_DIR}/openstack" - RSYNC_CMD="rsync --archive --safe-links --ignore-errors --quiet --no-perms --no-owner --no-group" - ${RSYNC_CMD} /var/log/ "${GATE_LOG_DIR}/host" || true - ${RSYNC_CMD} /openstack/log/ "${GATE_LOG_DIR}/openstack" || true + RSYNC_OPTS="--archive --safe-links --ignore-errors --quiet --no-perms --no-owner --no-group" + rsync $RSYNC_OPTS /var/log/ "${GATE_LOG_DIR}/host" || true + rsync $RSYNC_OPTS /openstack/log/ "${GATE_LOG_DIR}/openstack" || true # Rename all files gathered to have a .txt suffix so that the compressed # files are viewable via a web browser in OpenStack-CI. # except tempest results testrepository.subunit and testr_results.html @@ -167,7 +168,8 @@ function gate_job_exit_tasks { fi # Ensure that the files are readable by all users, including the non-root # OpenStack-CI jenkins user. - chmod -R 0777 "${GATE_LOG_DIR}" + chmod -R ugo+rX "${GATE_LOG_DIR}" + chown -R $(whoami) "${GATE_LOG_DIR}" fi } @@ -187,7 +189,10 @@ function run_dstat { ;; esac - dstat -tcmsdn --top-cpu --top-mem --top-bio --nocolor --output /openstack/log/instance-info/dstat.csv 3 > /openstack/log/instance-info/dstat.log& + # https://stackoverflow.com/a/20338327 executing in ()& decouples the dstat + # process from scripts-library to prevent hung builds if dstat fails to exit + # for any reason. + (dstat -tcmsdn --top-cpu --top-mem --top-bio --nocolor --output /openstack/log/instance-info/dstat.csv 3 > /openstack/log/instance-info/dstat.log)& } function generate_dstat_charts { diff --git a/scripts/test-log-collect.sh b/scripts/test-log-collect.sh new file mode 100755 index 0000000000..0dbe107a59 --- /dev/null +++ b/scripts/test-log-collect.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright 2017, Logan Vig +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Shell Opts ---------------------------------------------------------------- +set -e -u -x + +# Store the clone repo root location +export OSA_CLONE_DIR="$(readlink -f $(dirname ${0})/..)" + +## Functions ----------------------------------------------------------------- +info_block "Checking for required libraries." 2> /dev/null || source "${OSA_CLONE_DIR}/scripts/scripts-library.sh" + +## Main ---------------------------------------------------------------------- +gate_job_exit_tasks diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 60ba00bb72..a404891b47 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -21,6 +21,7 @@ Uses the gate-check-commit.sh script, running a default aio deploy. run: zuul.d/playbooks/run.yml + post-run: zuul.d/playbooks/post.yml timeout: 7200 irrelevant-files: - ^\.git.* diff --git a/zuul.d/playbooks/post.yml b/zuul.d/playbooks/post.yml new file mode 100644 index 0000000000..93113798d3 --- /dev/null +++ b/zuul.d/playbooks/post.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2017, Logan Vig +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: all + tasks: + - name: Run log collection script + command: scripts/test-log-collect.sh + become: yes + become_user: root + args: + chdir: "src/{{ zuul.project.canonical_name }}" + environment: + # ZUUL_PROJECT is used by the log collection functions to enable + # log collection configuration specific to OpenStack CI + ZUUL_PROJECT: "{{ zuul.project.short_name }}" + - name: Copy logs back to the executor + synchronize: + src: "src/{{ zuul.project.canonical_name }}/logs" + dest: "{{ zuul.executor.log_root }}" + mode: pull