Fix ARA/log data collection and reporting

Currently the ARA database is not collected after each
test, nor is the reporting done as it should be. This
patch does the following to resolve that:

1. Informs the log collection to collect to and from
   the right source folder.

2. Verifies that the logs directory exists to ensure
   that a missing directory does not cause the job to
   fail.

3. Makes the log collection quiet, so that only errors
   are reported when pulling them back to the zuul
   executor.

4. Removes a layer of indirection by making zuul use
   the scripts library directly instead of working
   through yet another wrapper script.

5. Sets the exit code properly so that the ARA report
   is run on test execution failure.

Backport note:
This is a re-implementation of the patch done to
master as this branch does not have the cross-repo
functionality applied.

Change-Id: Ie8da165a33180d03c840a69dae0aa5f424a69e50

scripts/scripts-library.sh

@@ -128,7 +128,12 @@ function gate_job_exit_tasks {
   # This environment variable captures the exit code
   # which was present when the trap was initiated.
   # This would be the success/failure of the test.
-  export TEST_EXIT_CODE=$?
+  TEST_EXIT_CODE=${TEST_EXIT_CODE:-$?}
+
+  # Specify a default location to capture logs into,
+  # just in case one is not provided (eg: when not run
+  # by zuul).
+  GATE_LOG_DIR=${GATE_LOG_DIR:-/opt/openstack-ansible/logs}
 
   # If this is a gate node from OpenStack-Infra Store all logs into the
   #  execution directory after gate run.
@@ -136,7 +141,7 @@ function gate_job_exit_tasks {
     if [ "$GATE_EXIT_RUN_DSTAT" == true ]; then
       generate_dstat_charts || true
     fi
-    GATE_LOG_DIR="${OSA_CLONE_DIR:-$(dirname $0)/..}/logs"
+
     mkdir -p "${GATE_LOG_DIR}/host" "${GATE_LOG_DIR}/openstack"
     RSYNC_OPTS="--archive --safe-links --ignore-errors --quiet --no-perms --no-owner --no-group"
     rsync $RSYNC_OPTS /var/log/ "${GATE_LOG_DIR}/host" || true
@@ -148,18 +153,26 @@ function gate_job_exit_tasks {
 
     # Generate the ARA report if enabled
     if [ "$GATE_EXIT_RUN_ARA" == true ]; then
+
+      # Define the ARA path for reusability
+      ARA_CMD="/opt/ansible-runtime/bin/ara"
+
+      # Create the ARA log directory and store the sqlite source database
+      mkdir ${GATE_LOG_DIR}/ara
+      rsync $RSYNC_OPTS "${HOME}/.ara/ansible.sqlite" "${GATE_LOG_DIR}/ara/"
+
       # In order to reduce the quantity of unnecessary log content
       # being kept in OpenStack-Infra we only generate the ARA report
       # when the test result is a failure.
-      if [[ "${TEST_EXIT_CODE}" != "0" ]]; then
+      if [[ "${TEST_EXIT_CODE}" != "0" ]] && [[ "${TEST_EXIT_CODE}" != "true" ]]; then
         echo "Generating ARA report due to non-zero exit code (${TEST_EXIT_CODE})."
-        /opt/ansible-runtime/bin/ara generate html "${GATE_LOG_DIR}/ara" || true
+        ${ARA_CMD} generate html "${GATE_LOG_DIR}/ara" || true
       else
         echo "Not generating ARA report due to test pass."
       fi
       # We still want the subunit report though, as that reflects
       # success/failure in OpenStack Health
-      /opt/ansible-runtime/bin/ara generate subunit "${GATE_LOG_DIR}/ara/testrepository.subunit" || true
+      ${ARA_CMD} generate subunit "${GATE_LOG_DIR}/ara/testrepository.subunit" || true
     fi
     # Compress the files gathered so that they do not take up too much space.
     # We use 'command' to ensure that we're not executing with some sort of alias.

scripts/test-log-collect.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017, Logan Vig <logan2211@gmail.com>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## Shell Opts ----------------------------------------------------------------
-set -e -u -x
-
-# Store the clone repo root location
-export OSA_CLONE_DIR="$(readlink -f $(dirname ${0})/..)"
-
-## Functions -----------------------------------------------------------------
-info_block "Checking for required libraries." 2> /dev/null || source "${OSA_CLONE_DIR}/scripts/scripts-library.sh"
-
-## Main ----------------------------------------------------------------------
-gate_job_exit_tasks

zuul.d/playbooks/post.yml

@@ -16,17 +16,37 @@
 - hosts: all
   tasks:
     - name: Run log collection script
-      command: scripts/test-log-collect.sh
+      shell: |
+        source scripts/scripts-library.sh
+        gate_job_exit_tasks
       become: yes
       become_user: root
       args:
-        chdir: "src/{{ zuul.project.canonical_name }}"
+        chdir: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}"
+        executable: /bin/bash
       environment:
         # ZUUL_PROJECT is used by the log collection functions to enable
         # log collection configuration specific to OpenStack CI
         ZUUL_PROJECT: "{{ zuul.project.short_name }}"
+        TEST_EXIT_CODE: "{{ zuul_success | lower }}"
+        GATE_LOG_DIR: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}/logs"
+
+    - name: Check whether a logs folder exists
+      stat:
+        path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}/logs"
+        get_attributes: no
+        get_checksum: no
+        get_md5: no
+        get_mime: no
+      register: logs_folder
+
     - name: Copy logs back to the executor
       synchronize:
-        src: "src/{{ zuul.project.canonical_name }}/logs"
-        dest: "{{ zuul.executor.log_root }}"
+        src: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}/logs"
+        dest: "{{ zuul.executor.log_root }}/"
         mode: pull
+        rsync_opts:
+          - "--quiet"
+      when:
+        - logs_folder.stat is defined
+        - logs_folder.stat.exists | bool