diff --git a/docs/tripleo-quickstart-logs.html b/docs/tripleo-quickstart-logs.html index e1873d4c5..918010c74 100644 --- a/docs/tripleo-quickstart-logs.html +++ b/docs/tripleo-quickstart-logs.html @@ -49,12 +49,39 @@ Quickstart itself.

+ +

Variables used in the job run

+

The logs contain files showing variables used in the job run.

+ + +

Dry Run option

+

As a debugging step, a job can be run manually with '-dryrun' +appended to the job name. When the "playbook dry run" option is invoked, +the playbooks will not execute and collect logs will not run but +certain log files, including 'toci_env_args_output.log', which +contains the environment variables used in the job, and +playbook_executions.log will still be produced in the logs +directory for inspection. This option serves to assist with +debugging and to test the testing scripts themselves.

diff --git a/scripts/oooq_common_functions.sh b/scripts/oooq_common_functions.sh index 75931a428..4d85ee757 100644 --- a/scripts/oooq_common_functions.sh +++ b/scripts/oooq_common_functions.sh @@ -88,3 +88,71 @@ function run_with_timeout { /usr/bin/timeout --preserve-status ${TIME_FOR_COMMAND}m ${COMMAND} } +function collect_logs { + cat <<-EOF > $LOGS_DIR/collect_logs.sh + #!/bin/bash + set -x + + export NODEPOOL_PROVIDER=${NODEPOOL_PROVIDER:-''} + export STATS_TESTENV=${STATS_TESTENV:-''} + export STATS_OOOQ=${STATS_OOOQ:-''} + export START_JOB_TIME=${START_JOB_TIME:-''} + export ZUUL_PIPELINE=${ZUUL_PIPELINE:-''} + export DEVSTACK_GATE_TIMEOUT=${DEVSTACK_GATE_TIMEOUT:-''} + export REMAINING_TIME=${REMAINING_TIME:-''} + export LOCAL_WORKING_DIR="$WORKSPACE/.quickstart" + export OPT_WORKDIR=$LOCAL_WORKING_DIR + export WORKING_DIR="$HOME" + export LOGS_DIR=$WORKSPACE/logs + export VIRTUAL_ENV_DISABLE_PROMPT=1 + export ANSIBLE_CONFIG=$OOOQ_DIR/ansible.cfg + export ARA_DATABASE=sqlite:///${LOCAL_WORKING_DIR}/ara.sqlite + export ZUUL_CHANGES=${ZUUL_CHANGES:-''} + export NODES_FILE=${NODES_FILE:-''} + export TOCI_JOBTYPE=$TOCI_JOBTYPE + export STABLE_RELEASE=${STABLE_RELEASE:-''} + export QUICKSTART_RELEASE=${QUICKSTART_RELEASE:-''} + + set +u + source $LOCAL_WORKING_DIR/bin/activate + set -u + source $OOOQ_DIR/ansible_ssh_env.sh + + /usr/bin/timeout --preserve-status 40m $QUICKSTART_COLLECTLOGS_CMD > $LOGS_DIR/quickstart_collect_logs.log || \ + echo "WARNING: quickstart collect-logs failed, check quickstart_collectlogs.log for details" + + cp $LOGS_DIR/undercloud/var/log/postci.txt.gz $LOGS_DIR/ || true + + if [[ -e $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ]]; then + cp $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz + elif [[ -e $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ]]; then + cp $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz + elif [[ -e $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ]]; then + cp $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz + fi + + # Copy tempest.html to root dir + cp $LOGS_DIR/undercloud/home/$USER/tempest/tempest.html.gz ${LOGS_DIR} || true + + # Copy tempest and .testrepository directory to /opt/stack/new/tempest and + # unzip + sudo mkdir -p /opt/stack/new + sudo cp -Rf $LOGS_DIR/undercloud/home/$USER/tempest /opt/stack/new || true + sudo gzip -d -r /opt/stack/new/tempest/.testrepository || true + + # record the size of the logs directory + # -L, --dereference dereference all symbolic links + # Note: tail -n +1 is to prevent the error 'Broken Pipe' e.g. 'sort: write failed: standard output: Broken pipe' + + du -L -ch $LOGS_DIR/* | tail -n +1 | sort -rh | head -n 200 &> $LOGS_DIR/log-size.txt || true +EOF + + if [[ "${NODEPOOL_PROVIDER:-''}" == "rdo-cloud-tripleo" ]] || [[ "${NODEPOOL_PROVIDER:-''}" == "tripleo-test-cloud-rh1" ]]; then + if [[ "$TOCI_JOBTYPE" =~ "ovb" ]]; then + bash $LOGS_DIR/collect_logs.sh + # rename script to not to run it in multinode jobs + mv $LOGS_DIR/collect_logs.sh $LOGS_DIR/ovb_collect_logs.sh + fi + fi + +} diff --git a/toci_gate_test-oooq.sh b/toci_gate_test-oooq.sh index 2bbbd5ab1..1ac65f96a 100755 --- a/toci_gate_test-oooq.sh +++ b/toci_gate_test-oooq.sh @@ -117,6 +117,8 @@ TIMEOUT_SECS=$((DEVSTACK_GATE_TIMEOUT*60)) export EXTRA_VARS=${EXTRA_VARS:-""} export NODES_ARGS="" export EXTRANODE="" +# Set playbook execution status +export PLAYBOOK_DRY_RUN=${PLAYBOOK_DRY_RUN:=0} export COLLECT_CONF="$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/collect-logs.yml" LOCAL_WORKING_DIR="$WORKSPACE/.quickstart" LWD=$LOCAL_WORKING_DIR @@ -221,6 +223,9 @@ for JOB_TYPE_PART in $(sed 's/-/ /g' <<< "${TOCI_JOBTYPE:-}") ; do ;; gate) ;; + dryrun) + PLAYBOOK_DRY_RUN=1 + ;; *) # the rest should be node configuration NODES_FILE="$TRIPLEO_ROOT/tripleo-quickstart/config/nodes/$JOB_TYPE_PART.yml" diff --git a/toci_quickstart.sh b/toci_quickstart.sh index 5cdb972c6..bb46d4bd4 100755 --- a/toci_quickstart.sh +++ b/toci_quickstart.sh @@ -121,94 +121,55 @@ set -u source $OOOQ_DIR/ansible_ssh_env.sh [[ -n ${STATS_OOOQ:-''} ]] && export STATS_OOOQ=$(( $(date +%s) - STATS_OOOQ )) - - -for playbook in $PLAYBOOKS; do - echo "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" - run_with_timeout $START_JOB_TIME $QUICKSTART_INSTALL_CMD \ - "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" \ - $NODES_ARGS \ - $FEATURESET_CONF \ - $ENV_VARS \ - $EXTRA_VARS \ - $DEFAULT_ARGS \ - --extra-vars ci_job_end_time=$(( START_JOB_TIME + REMAINING_TIME*60 )) \ - $LOCAL_WORKING_DIR/playbooks/$playbook "${PLAYBOOKS_ARGS[$playbook]:-}" \ - 2>&1 | tee -a $LOGS_DIR/quickstart_install.log && exit_value=0 || exit_value=$? - - # Print status of playbook run - [[ "$exit_value" == 0 ]] && echo "Playbook run of $playbook passed successfully" - [[ "$exit_value" != 0 ]] && echo "Playbook run of $playbook failed" && break -done - -[[ "$exit_value" == 0 ]] && echo "Playbook run passed successfully" || echo "Playbook run failed" - -## LOGS COLLECTION - -cat < $LOGS_DIR/collect_logs.sh -#!/bin/bash -set -x - -export NODEPOOL_PROVIDER=${NODEPOOL_PROVIDER:-''} -export STATS_TESTENV=${STATS_TESTENV:-''} -export STATS_OOOQ=${STATS_OOOQ:-''} -export START_JOB_TIME=${START_JOB_TIME:-''} -export ZUUL_PIPELINE=${ZUUL_PIPELINE:-''} -export DEVSTACK_GATE_TIMEOUT=${DEVSTACK_GATE_TIMEOUT:-''} -export REMAINING_TIME=${REMAINING_TIME:-''} -export LOCAL_WORKING_DIR="$WORKSPACE/.quickstart" -export OPT_WORKDIR=$LOCAL_WORKING_DIR -export WORKING_DIR="$HOME" -export LOGS_DIR=$WORKSPACE/logs -export VIRTUAL_ENV_DISABLE_PROMPT=1 -export ANSIBLE_CONFIG=$OOOQ_DIR/ansible.cfg -export ARA_DATABASE=sqlite:///${LOCAL_WORKING_DIR}/ara.sqlite -export ZUUL_CHANGES=${ZUUL_CHANGES:-''} -export NODES_FILE=${NODES_FILE:-''} -export TOCI_JOBTYPE=$TOCI_JOBTYPE -export STABLE_RELEASE=${STABLE_RELEASE:-''} -export QUICKSTART_RELEASE=${QUICKSTART_RELEASE:-''} - -set +u -source $LOCAL_WORKING_DIR/bin/activate -set -u -source $OOOQ_DIR/ansible_ssh_env.sh - -/usr/bin/timeout --preserve-status 40m $QUICKSTART_COLLECTLOGS_CMD > $LOGS_DIR/quickstart_collect_logs.log || \ - echo "WARNING: quickstart collect-logs failed, check quickstart_collectlogs.log for details" - -cp $LOGS_DIR/undercloud/var/log/postci.txt.gz $LOGS_DIR/ || true - -if [[ -e $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ]]; then - cp $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz -elif [[ -e $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ]]; then - cp $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz -elif [[ -e $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ]]; then - cp $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz +# Debug step capture env variables +if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then + echo "-- Capture Environment Variables Used ---------" + echo "$(env)" | tee -a $LOGS_DIR/toci_env_args_output.log + declare -p | tee -a $LOGS_DIR/toci_env_args_output.log fi -# Copy tempest.html to root dir -cp $LOGS_DIR/undercloud/home/$USER/tempest/tempest.html.gz ${LOGS_DIR} || true +echo "-- Playbooks Output --------------------------" +for playbook in $PLAYBOOKS; do + echo "$QUICKSTART_INSTALL_CMD \ + ${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG} \ + $NODES_ARGS \ + $FEATURESET_CONF \ + $ENV_VARS \ + $EXTRA_VARS \ + $DEFAULT_ARGS \ + $LOCAL_WORKING_DIR/playbooks/$playbook ${PLAYBOOKS_ARGS[$playbook]:-}" \ + | sed 's/--/\n--/g' \ + | tee -a $LOGS_DIR/playbook_executions.log + echo "# --------------------------------------- " \ + | tee -a $LOGS_DIR/playbook_executions.log +done -# Copy tempest and .testrepository directory to /opt/stack/new/tempest and -# unzip -sudo mkdir -p /opt/stack/new -sudo cp -Rf $LOGS_DIR/undercloud/home/$USER/tempest /opt/stack/new || true -sudo gzip -d -r /opt/stack/new/tempest/.testrepository || true +if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then + exit_value=0 +else + for playbook in $PLAYBOOKS; do + echo "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" + run_with_timeout $START_JOB_TIME $QUICKSTART_INSTALL_CMD \ + "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" \ + $NODES_ARGS \ + $FEATURESET_CONF \ + $ENV_VARS \ + $EXTRA_VARS \ + $DEFAULT_ARGS \ + --extra-vars ci_job_end_time=$(( START_JOB_TIME + REMAINING_TIME*60 )) \ + $LOCAL_WORKING_DIR/playbooks/$playbook "${PLAYBOOKS_ARGS[$playbook]:-}" \ + 2>&1 | tee -a $LOGS_DIR/quickstart_install.log && exit_value=0 || exit_value=$? -# record the size of the logs directory -# -L, --dereference dereference all symbolic links -# Note: tail -n +1 is to prevent the error 'Broken Pipe' e.g. 'sort: write failed: standard output: Broken pipe' + # Print status of playbook run + [[ "$exit_value" == 0 ]] && echo "Playbook run of $playbook passed successfully" + [[ "$exit_value" != 0 ]] && echo "Playbook run of $playbook failed" && break + done -du -L -ch $LOGS_DIR/* | tail -n +1 | sort -rh | head -n 200 &> $LOGS_DIR/log-size.txt || true -EOF + [[ "$exit_value" == 0 ]] && echo "Playbook run passed successfully" || echo "Playbook run failed" + + ## LOGS COLLECTION + collect_logs -if [[ "${NODEPOOL_PROVIDER:-''}" == "rdo-cloud-tripleo" ]] || [[ "${NODEPOOL_PROVIDER:-''}" == "tripleo-test-cloud-rh1" ]]; then - if [[ "$TOCI_JOBTYPE" =~ "ovb" ]]; then - bash $LOGS_DIR/collect_logs.sh - # rename script to not to run it in multinode jobs - mv $LOGS_DIR/collect_logs.sh $LOGS_DIR/ovb_collect_logs.sh - fi fi popd