Collect logs in post playbook for multinode

Collect logs after main job is finished, so that we'll have logs
in any case of job result.
Do it for multinode jobs only, because ovb nodes are wiped right
after job finishes and collecting logs there requires a bit
different effort.
Depends-On: I7b7582469b01116bbe754af07c81cc698355d8c4

Change-Id: I600ade65052d28978c9d395323c8b86ed213fd38

playbooks/tripleo-ci/post.yaml

@@ -1,3 +1,13 @@
+- name: Collect logs
+  hosts: primary
+  tasks:
+
+    - name: Run ansible playbook to collect logs
+      shell: |
+        if [[ -e {{ ansible_user_dir }}/workspace/logs/collect_logs.sh ]]; then
+          bash {{ ansible_user_dir }}/workspace/logs/collect_logs.sh
+        fi
+
 - hosts:
     - primary
     - tripleo-centos-7

toci_gate_test-oooq.sh

@@ -154,10 +154,7 @@ for JOB_TYPE_PART in $(sed 's/-/ /g' <<< "${TOCI_JOBTYPE:-}") ; do
             SUBNODES_SSH_KEY=/etc/nodepool/id_rsa
             ENVIRONMENT="osinfra"
             PLAYBOOK="multinode.yml"
-            FEATURESET_CONF="
-                --extra-vars @$LWD/config/general_config/featureset-multinode-common.yml
-                $FEATURESET_CONF
-            "
+            FEATURESET_CONF=" --extra-vars @$LWD/config/general_config/featureset-multinode-common.yml $FEATURESET_CONF"
             if [[ $NODEPOOL_PROVIDER == "rdo-cloud-tripleo" ]]; then
                 ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode-rdocloud.yml"
             else
@@ -172,10 +169,7 @@ for JOB_TYPE_PART in $(sed 's/-/ /g' <<< "${TOCI_JOBTYPE:-}") ; do
             ENVIRONMENT="osinfra"
             UNDERCLOUD="127.0.0.2"
             PLAYBOOK="multinode.yml"
-            FEATURESET_CONF="
-                --extra-vars @$LWD/config/general_config/featureset-multinode-common.yml
-                $FEATURESET_CONF
-            "
+            FEATURESET_CONF=" --extra-vars @$LWD/config/general_config/featureset-multinode-common.yml $FEATURESET_CONF"
             if [[ $NODEPOOL_PROVIDER == "rdo-cloud-tripleo" ]]; then
                 ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode-rdocloud.yml"
             else

toci_quickstart.sh

@@ -12,13 +12,12 @@ source $TRIPLEO_ROOT/tripleo-ci/scripts/oooq_common_functions.sh
 ## Signal to toci_gate_test.sh we've started by
 touch /tmp/toci.started
 
-export DEFAULT_ARGS="
-    --extra-vars local_working_dir=$LOCAL_WORKING_DIR
-    --extra-vars virthost=$UNDERCLOUD
-    --inventory $LOCAL_WORKING_DIR/hosts
-    --extra-vars tripleo_root=$TRIPLEO_ROOT
-    --extra-vars working_dir=$WORKING_DIR
-    --extra-vars validation_args='--validation-errors-nonfatal'
+export DEFAULT_ARGS="--extra-vars local_working_dir=$LOCAL_WORKING_DIR \
+    --extra-vars virthost=$UNDERCLOUD \
+    --inventory $LOCAL_WORKING_DIR/hosts \
+    --extra-vars tripleo_root=$TRIPLEO_ROOT \
+    --extra-vars working_dir=$WORKING_DIR \
+    --extra-vars validation_args='--validation-errors-nonfatal' \
 "
 
 # --install-deps arguments installs deps and then quits, no other arguments are
@@ -51,19 +50,18 @@ QUICKSTART_INSTALL_CMD="
     --skip-tags teardown-all
 "
 
-QUICKSTART_COLLECTLOGS_CMD="
-    $LOCAL_WORKING_DIR/bin/ansible-playbook
-    $LOCAL_WORKING_DIR/playbooks/collect-logs.yml
-    --extra-vars @$LOCAL_WORKING_DIR/config/release/tripleo-ci/$QUICKSTART_RELEASE.yml
-    $NODES_ARGS
-    $FEATURESET_CONF
-    $ENV_VARS
-    $EXTRA_VARS
-    $DEFAULT_ARGS
-    --extra-vars @$COLLECT_CONF
-    --extra-vars artcl_collect_dir=$LOGS_DIR
-    --tags all
-    --skip-tags teardown-all
+QUICKSTART_COLLECTLOGS_CMD="$LOCAL_WORKING_DIR/bin/ansible-playbook \
+    $LOCAL_WORKING_DIR/playbooks/collect-logs.yml \
+    -vv \
+    --extra-vars @$LOCAL_WORKING_DIR/config/release/tripleo-ci/$QUICKSTART_RELEASE.yml \
+    $FEATURESET_CONF \
+    $ENV_VARS \
+    $EXTRA_VARS \
+    $DEFAULT_ARGS \
+    --extra-vars @$COLLECT_CONF \
+    --extra-vars artcl_collect_dir=$LOGS_DIR \
+    --tags all \
+    --skip-tags teardown-all \
 "
 
 mkdir -p $LOCAL_WORKING_DIR
@@ -96,11 +94,25 @@ run_with_timeout $START_JOB_TIME $QUICKSTART_INSTALL_CMD --extra-vars ci_job_end
 [[ "$exit_value" == 0 ]] && echo "Playbook run passed successfully" || echo "Playbook run failed"
 ## LOGS COLLECTION
 
-$QUICKSTART_COLLECTLOGS_CMD \
-    > $LOGS_DIR/quickstart_collect_logs.log || \
+cat <<EOF > $LOGS_DIR/collect_logs.sh
+#!/bin/bash
+set -x
+
+export LOCAL_WORKING_DIR="$WORKSPACE/.quickstart"
+export OPT_WORKDIR=$LOCAL_WORKING_DIR
+export WORKING_DIR="$HOME"
+export LOGS_DIR=$WORKSPACE/logs
+export VIRTUAL_ENV_DISABLE_PROMPT=1
+export ANSIBLE_CONFIG=$OOOQ_DIR/ansible.cfg
+export ARA_DATABASE=sqlite:///${LOCAL_WORKING_DIR}/ara.sqlite
+set +u
+source $LOCAL_WORKING_DIR/bin/activate
+set -u
+source $OOOQ_DIR/ansible_ssh_env.sh
+
+$QUICKSTART_COLLECTLOGS_CMD  > $LOGS_DIR/quickstart_collect_logs.log || \
     echo "WARNING: quickstart collect-logs failed, check quickstart_collectlogs.log for details"
 
-# Temporary workaround to make postci log visible as it was before
 cp $LOGS_DIR/undercloud/var/log/postci.txt.gz $LOGS_DIR/ || true
 
 if [[ -e $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ]]; then
@@ -120,6 +132,19 @@ sudo mkdir -p /opt/stack/new
 sudo cp -Rf $LOGS_DIR/undercloud/home/$USER/tempest /opt/stack/new || true
 sudo gzip -d -r /opt/stack/new/tempest/.testrepository || true
 
+# record the size of the logs directory
+# -L, --dereference     dereference all symbolic links
+# Note: tail -n +1 is to prevent the error 'Broken Pipe' e.g. 'sort: write failed: standard output: Broken pipe'
+
+du -L -ch $LOGS_DIR/* | tail -n +1 | sort -rh | head -n 200 &> $LOGS_DIR/log-size.txt || true
+EOF
+
+if [ ${NODEPOOL_PROVIDER:-''} == 'rdo-cloud-tripleo' ] || [ ${NODEPOOL_PROVIDER:-''} =='tripleo-test-cloud-rh1' ]; then
+    bash $LOGS_DIR/collect_logs.sh
+    # rename script to not to run it in multinode jobs
+    mv $LOGS_DIR/collect_logs.sh $LOGS_DIR/ovb_collect_logs.sh
+fi
+
 export ARA_DATABASE="sqlite:///$LOCAL_WORKING_DIR/ara.sqlite"
 $LOCAL_WORKING_DIR/bin/ara generate html $LOGS_DIR/ara_oooq || true
 gzip --best --recursive $LOGS_DIR/ara_oooq || true
@@ -129,12 +154,6 @@ sudo unbound-control dump_cache > /tmp/dns_cache.txt
 sudo chown ${USER}: /tmp/dns_cache.txt
 cat /tmp/dns_cache.txt | gzip - > $LOGS_DIR/dns_cache.txt.gz
 
-
-# record the size of the logs directory
-# -L, --dereference     dereference all symbolic links
-# Note: tail -n +1 is to prevent the error "Broken Pipe" e.g. "sort: write failed: standard output: Broken pipe"
-du -L -ch $LOGS_DIR/* | tail -n +1 | sort -rh | head -n 200 &> $LOGS_DIR/log-size.txt || true
-
 if [[ "$PERIODIC" == 1 && -e $WORKSPACE/hash_info.sh ]] ; then
     echo export JOB_EXIT_VALUE=$exit_value >> $WORKSPACE/hash_info.sh
 fi