From 3dab6f0df04aa45c18a43526e15baad428130df7 Mon Sep 17 00:00:00 2001 From: Ronelle Landy Date: Sun, 27 May 2018 17:53:11 -0400 Subject: [PATCH] Add dry run option to toci_quickstart - playbook order in tact With dry run, playbooks are not executed. The env variables, releases script output and playbooks commands are written out to files for debug. The review also moves the collect logs lines to a function for easier organization Documentation for the dry run option and variable files is added. Change-Id: I0bad5ee3150d94115bed018de9488590305a2b80 --- docs/tripleo-quickstart-logs.html | 29 ++++++- scripts/oooq_common_functions.sh | 68 ++++++++++++++++ toci_gate_test-oooq.sh | 5 ++ toci_quickstart.sh | 125 ++++++++++-------------------- 4 files changed, 144 insertions(+), 83 deletions(-) diff --git a/docs/tripleo-quickstart-logs.html b/docs/tripleo-quickstart-logs.html index e1873d4c5..918010c74 100644 --- a/docs/tripleo-quickstart-logs.html +++ b/docs/tripleo-quickstart-logs.html @@ -49,12 +49,39 @@ Quickstart itself.

+ +

Variables used in the job run

+

The logs contain files showing variables used in the job run.

+ + +

Dry Run option

+

As a debugging step, a job can be run manually with '-dryrun' +appended to the job name. When the "playbook dry run" option is invoked, +the playbooks will not execute and collect logs will not run but +certain log files, including 'toci_env_args_output.log', which +contains the environment variables used in the job, and +playbook_executions.log will still be produced in the logs +directory for inspection. This option serves to assist with +debugging and to test the testing scripts themselves.

diff --git a/scripts/oooq_common_functions.sh b/scripts/oooq_common_functions.sh index 75931a428..4d85ee757 100644 --- a/scripts/oooq_common_functions.sh +++ b/scripts/oooq_common_functions.sh @@ -88,3 +88,71 @@ function run_with_timeout { /usr/bin/timeout --preserve-status ${TIME_FOR_COMMAND}m ${COMMAND} } +function collect_logs { + cat <<-EOF > $LOGS_DIR/collect_logs.sh + #!/bin/bash + set -x + + export NODEPOOL_PROVIDER=${NODEPOOL_PROVIDER:-''} + export STATS_TESTENV=${STATS_TESTENV:-''} + export STATS_OOOQ=${STATS_OOOQ:-''} + export START_JOB_TIME=${START_JOB_TIME:-''} + export ZUUL_PIPELINE=${ZUUL_PIPELINE:-''} + export DEVSTACK_GATE_TIMEOUT=${DEVSTACK_GATE_TIMEOUT:-''} + export REMAINING_TIME=${REMAINING_TIME:-''} + export LOCAL_WORKING_DIR="$WORKSPACE/.quickstart" + export OPT_WORKDIR=$LOCAL_WORKING_DIR + export WORKING_DIR="$HOME" + export LOGS_DIR=$WORKSPACE/logs + export VIRTUAL_ENV_DISABLE_PROMPT=1 + export ANSIBLE_CONFIG=$OOOQ_DIR/ansible.cfg + export ARA_DATABASE=sqlite:///${LOCAL_WORKING_DIR}/ara.sqlite + export ZUUL_CHANGES=${ZUUL_CHANGES:-''} + export NODES_FILE=${NODES_FILE:-''} + export TOCI_JOBTYPE=$TOCI_JOBTYPE + export STABLE_RELEASE=${STABLE_RELEASE:-''} + export QUICKSTART_RELEASE=${QUICKSTART_RELEASE:-''} + + set +u + source $LOCAL_WORKING_DIR/bin/activate + set -u + source $OOOQ_DIR/ansible_ssh_env.sh + + /usr/bin/timeout --preserve-status 40m $QUICKSTART_COLLECTLOGS_CMD > $LOGS_DIR/quickstart_collect_logs.log || \ + echo "WARNING: quickstart collect-logs failed, check quickstart_collectlogs.log for details" + + cp $LOGS_DIR/undercloud/var/log/postci.txt.gz $LOGS_DIR/ || true + + if [[ -e $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ]]; then + cp $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz + elif [[ -e $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ]]; then + cp $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz + elif [[ -e $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ]]; then + cp $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz + fi + + # Copy tempest.html to root dir + cp $LOGS_DIR/undercloud/home/$USER/tempest/tempest.html.gz ${LOGS_DIR} || true + + # Copy tempest and .testrepository directory to /opt/stack/new/tempest and + # unzip + sudo mkdir -p /opt/stack/new + sudo cp -Rf $LOGS_DIR/undercloud/home/$USER/tempest /opt/stack/new || true + sudo gzip -d -r /opt/stack/new/tempest/.testrepository || true + + # record the size of the logs directory + # -L, --dereference dereference all symbolic links + # Note: tail -n +1 is to prevent the error 'Broken Pipe' e.g. 'sort: write failed: standard output: Broken pipe' + + du -L -ch $LOGS_DIR/* | tail -n +1 | sort -rh | head -n 200 &> $LOGS_DIR/log-size.txt || true +EOF + + if [[ "${NODEPOOL_PROVIDER:-''}" == "rdo-cloud-tripleo" ]] || [[ "${NODEPOOL_PROVIDER:-''}" == "tripleo-test-cloud-rh1" ]]; then + if [[ "$TOCI_JOBTYPE" =~ "ovb" ]]; then + bash $LOGS_DIR/collect_logs.sh + # rename script to not to run it in multinode jobs + mv $LOGS_DIR/collect_logs.sh $LOGS_DIR/ovb_collect_logs.sh + fi + fi + +} diff --git a/toci_gate_test-oooq.sh b/toci_gate_test-oooq.sh index 2bbbd5ab1..1ac65f96a 100755 --- a/toci_gate_test-oooq.sh +++ b/toci_gate_test-oooq.sh @@ -117,6 +117,8 @@ TIMEOUT_SECS=$((DEVSTACK_GATE_TIMEOUT*60)) export EXTRA_VARS=${EXTRA_VARS:-""} export NODES_ARGS="" export EXTRANODE="" +# Set playbook execution status +export PLAYBOOK_DRY_RUN=${PLAYBOOK_DRY_RUN:=0} export COLLECT_CONF="$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/collect-logs.yml" LOCAL_WORKING_DIR="$WORKSPACE/.quickstart" LWD=$LOCAL_WORKING_DIR @@ -221,6 +223,9 @@ for JOB_TYPE_PART in $(sed 's/-/ /g' <<< "${TOCI_JOBTYPE:-}") ; do ;; gate) ;; + dryrun) + PLAYBOOK_DRY_RUN=1 + ;; *) # the rest should be node configuration NODES_FILE="$TRIPLEO_ROOT/tripleo-quickstart/config/nodes/$JOB_TYPE_PART.yml" diff --git a/toci_quickstart.sh b/toci_quickstart.sh index 5cdb972c6..bb46d4bd4 100755 --- a/toci_quickstart.sh +++ b/toci_quickstart.sh @@ -121,94 +121,55 @@ set -u source $OOOQ_DIR/ansible_ssh_env.sh [[ -n ${STATS_OOOQ:-''} ]] && export STATS_OOOQ=$(( $(date +%s) - STATS_OOOQ )) - - -for playbook in $PLAYBOOKS; do - echo "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" - run_with_timeout $START_JOB_TIME $QUICKSTART_INSTALL_CMD \ - "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" \ - $NODES_ARGS \ - $FEATURESET_CONF \ - $ENV_VARS \ - $EXTRA_VARS \ - $DEFAULT_ARGS \ - --extra-vars ci_job_end_time=$(( START_JOB_TIME + REMAINING_TIME*60 )) \ - $LOCAL_WORKING_DIR/playbooks/$playbook "${PLAYBOOKS_ARGS[$playbook]:-}" \ - 2>&1 | tee -a $LOGS_DIR/quickstart_install.log && exit_value=0 || exit_value=$? - - # Print status of playbook run - [[ "$exit_value" == 0 ]] && echo "Playbook run of $playbook passed successfully" - [[ "$exit_value" != 0 ]] && echo "Playbook run of $playbook failed" && break -done - -[[ "$exit_value" == 0 ]] && echo "Playbook run passed successfully" || echo "Playbook run failed" - -## LOGS COLLECTION - -cat < $LOGS_DIR/collect_logs.sh -#!/bin/bash -set -x - -export NODEPOOL_PROVIDER=${NODEPOOL_PROVIDER:-''} -export STATS_TESTENV=${STATS_TESTENV:-''} -export STATS_OOOQ=${STATS_OOOQ:-''} -export START_JOB_TIME=${START_JOB_TIME:-''} -export ZUUL_PIPELINE=${ZUUL_PIPELINE:-''} -export DEVSTACK_GATE_TIMEOUT=${DEVSTACK_GATE_TIMEOUT:-''} -export REMAINING_TIME=${REMAINING_TIME:-''} -export LOCAL_WORKING_DIR="$WORKSPACE/.quickstart" -export OPT_WORKDIR=$LOCAL_WORKING_DIR -export WORKING_DIR="$HOME" -export LOGS_DIR=$WORKSPACE/logs -export VIRTUAL_ENV_DISABLE_PROMPT=1 -export ANSIBLE_CONFIG=$OOOQ_DIR/ansible.cfg -export ARA_DATABASE=sqlite:///${LOCAL_WORKING_DIR}/ara.sqlite -export ZUUL_CHANGES=${ZUUL_CHANGES:-''} -export NODES_FILE=${NODES_FILE:-''} -export TOCI_JOBTYPE=$TOCI_JOBTYPE -export STABLE_RELEASE=${STABLE_RELEASE:-''} -export QUICKSTART_RELEASE=${QUICKSTART_RELEASE:-''} - -set +u -source $LOCAL_WORKING_DIR/bin/activate -set -u -source $OOOQ_DIR/ansible_ssh_env.sh - -/usr/bin/timeout --preserve-status 40m $QUICKSTART_COLLECTLOGS_CMD > $LOGS_DIR/quickstart_collect_logs.log || \ - echo "WARNING: quickstart collect-logs failed, check quickstart_collectlogs.log for details" - -cp $LOGS_DIR/undercloud/var/log/postci.txt.gz $LOGS_DIR/ || true - -if [[ -e $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ]]; then - cp $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz -elif [[ -e $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ]]; then - cp $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz -elif [[ -e $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ]]; then - cp $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz +# Debug step capture env variables +if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then + echo "-- Capture Environment Variables Used ---------" + echo "$(env)" | tee -a $LOGS_DIR/toci_env_args_output.log + declare -p | tee -a $LOGS_DIR/toci_env_args_output.log fi -# Copy tempest.html to root dir -cp $LOGS_DIR/undercloud/home/$USER/tempest/tempest.html.gz ${LOGS_DIR} || true +echo "-- Playbooks Output --------------------------" +for playbook in $PLAYBOOKS; do + echo "$QUICKSTART_INSTALL_CMD \ + ${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG} \ + $NODES_ARGS \ + $FEATURESET_CONF \ + $ENV_VARS \ + $EXTRA_VARS \ + $DEFAULT_ARGS \ + $LOCAL_WORKING_DIR/playbooks/$playbook ${PLAYBOOKS_ARGS[$playbook]:-}" \ + | sed 's/--/\n--/g' \ + | tee -a $LOGS_DIR/playbook_executions.log + echo "# --------------------------------------- " \ + | tee -a $LOGS_DIR/playbook_executions.log +done -# Copy tempest and .testrepository directory to /opt/stack/new/tempest and -# unzip -sudo mkdir -p /opt/stack/new -sudo cp -Rf $LOGS_DIR/undercloud/home/$USER/tempest /opt/stack/new || true -sudo gzip -d -r /opt/stack/new/tempest/.testrepository || true +if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then + exit_value=0 +else + for playbook in $PLAYBOOKS; do + echo "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" + run_with_timeout $START_JOB_TIME $QUICKSTART_INSTALL_CMD \ + "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" \ + $NODES_ARGS \ + $FEATURESET_CONF \ + $ENV_VARS \ + $EXTRA_VARS \ + $DEFAULT_ARGS \ + --extra-vars ci_job_end_time=$(( START_JOB_TIME + REMAINING_TIME*60 )) \ + $LOCAL_WORKING_DIR/playbooks/$playbook "${PLAYBOOKS_ARGS[$playbook]:-}" \ + 2>&1 | tee -a $LOGS_DIR/quickstart_install.log && exit_value=0 || exit_value=$? -# record the size of the logs directory -# -L, --dereference dereference all symbolic links -# Note: tail -n +1 is to prevent the error 'Broken Pipe' e.g. 'sort: write failed: standard output: Broken pipe' + # Print status of playbook run + [[ "$exit_value" == 0 ]] && echo "Playbook run of $playbook passed successfully" + [[ "$exit_value" != 0 ]] && echo "Playbook run of $playbook failed" && break + done -du -L -ch $LOGS_DIR/* | tail -n +1 | sort -rh | head -n 200 &> $LOGS_DIR/log-size.txt || true -EOF + [[ "$exit_value" == 0 ]] && echo "Playbook run passed successfully" || echo "Playbook run failed" + + ## LOGS COLLECTION + collect_logs -if [[ "${NODEPOOL_PROVIDER:-''}" == "rdo-cloud-tripleo" ]] || [[ "${NODEPOOL_PROVIDER:-''}" == "tripleo-test-cloud-rh1" ]]; then - if [[ "$TOCI_JOBTYPE" =~ "ovb" ]]; then - bash $LOGS_DIR/collect_logs.sh - # rename script to not to run it in multinode jobs - mv $LOGS_DIR/collect_logs.sh $LOGS_DIR/ovb_collect_logs.sh - fi fi popd