Produce separate named report files

Refactor role with tags for execute pre-run, run and
post-run phases.

Change-Id: I7daff63be56d2746a63e00d21aaa78078e60f60f

.gitignore

@@ -25,8 +25,7 @@ cover/
 doc/build/*
 dist/
 etc/*.sample
-last.subunit
+tobiko_results*
-tobiko_results.*
 zuul/versioninfo
 
 # Files created by releasenotes build

playbooks/tobiko-devstack-post-run.yaml

@@ -14,9 +14,18 @@
 
 ---
 
+- hosts: tempest
+  roles:
+    - role: tobiko
+      tags:
+        - tobiko-post-run
+      vars:
+        tobiko_pre_run: false
+        tobiko_run: false
+        tobiko_post_run: true
+
+
 - hosts: tempest
   become: true
   roles:
-    - role: fetch-subunit-output
-      zuul_work_dir: '{{ devstack_base_dir }}/tobiko'
     - role: process-stackviz

playbooks/tobiko-devstack-pre-run.yaml

@@ -29,18 +29,20 @@
       become: yes
 
 
-- name: Copy the build ssh key to authorized_keys on all hosts for stack
+- hosts: all
-  hosts: all
   roles:
     - role: copy-build-sshkey
       copy_sshkey_target_user: stack
-
-
-- hosts: all
-  roles:
     - multi-node-setup
-
-
-- hosts: all
-  roles:
     - orchestrate-devstack
+
+
+- hosts: tempest
+  roles:
+    - role: tobiko
+      tags:
+        - tobiko-pre-run
+      vars:
+        tobiko_pre_run: true
+        tobiko_run: false
+        tobiko_post_run: false

playbooks/tobiko-devstack-run.yaml

@@ -14,7 +14,10 @@
 
 ---
 
-- name: Run Tobiko
+- hosts: tempest
-  hosts: tempest
   roles:
     - role: tobiko
+      vars:
+        tobiko_pre_run: false
+        tobiko_run: true
+        tobiko_post_run: false

report/.gitignore

@@ -1 +0,0 @@
-/build/

report/build.sh

@@ -1,22 +0,0 @@
-#!/bin/sh
-
-# Produce test HTML report file into ${REPORT_DIR}
-
-set -xeu
-
-
-TESTR_DIR=${TESTR_DIR:-$(pwd)}
-BUILD_DIR=${BUILD_DIR:-${TESTR_DIR}}
-SUBUNIT_FILE=${SUBUNIT_FILE:-${BUILD_DIR}/last.subunit}
-TESTR_RESULTS_HTML=${TESTR_RESULTS_HTML:-${BUILD_DIR}/tobiko_results.html}
-TESTR_RESULTS_XML=${TESTR_RESULTS_XML:-${BUILD_DIR}/tobiko_results.xml}
-
-
-make_testr_results_html() {
-    mkdir -p "$(dirname ${SUBUNIT_FILE})"
-    (cd "${TESTR_DIR}" && stestr last --subunit) > "${SUBUNIT_FILE}"
-    (cd "${BUILD_DIR}" && subunit2html "${SUBUNIT_FILE}" "${TESTR_RESULTS_HTML}")
-    (cd "${BUILD_DIR}" && subunit2junitxml "${SUBUNIT_FILE}" -o "${TESTR_RESULTS_XML}")
-}
-
-make_testr_results_html

roles/tobiko/defaults/main.yaml

@@ -1,10 +1,11 @@
 ---
 
-# Tobiko workflow parameters
+# Tobiko workflow flags
-tobiko_configure: true
+tobiko_pre_run: true
-tobiko_check_tools: true
+tobiko_run: "{{ tobiko_run_tests or tobiko_run_faults }}"
 tobiko_run_tests: true
 tobiko_run_faults: false
+tobiko_post_run: true
 
 
 # Tobiko sources tobiko directory
@@ -12,10 +13,6 @@ tobiko_source_dir: "{{ role_path | dirname | dirname }}"
 tobiko_dir: "{{ ansible_env.HOME }}/src/tobiko"
 tobiko_user: "{{ ansible_user }}"
 
-# Logging options
-tobiko_debug: True
-tobiko_log_file: "{{ tobiko_dir }}/tobiko.log"
-
 
 # Configuration options
 tobiko_conf_file: /etc/tobiko/tobiko.conf
@@ -34,10 +31,16 @@ tobiko_conf_options:
     value: "{{ tobiko_log_file | basename }}"
 
 
+# Logging options
+tobiko_debug: True
+tobiko_log_file: "{{ tobiko_dir }}/tobiko.log"
+
+
 # Output directories where tobiko should write to
 tobiko_output_dirs:
   - "{{ tox_dir }}"
   - "{{ tobiko_log_file | dirname }}"
+  - "{{ test_report_dir }}"
 
 
 tobiko_become_user: ''
@@ -54,3 +57,10 @@ tox_command: "{{ tobiko_dir }}/tools/ci/tox"
 tox_envlist: scenario
 tox_extra_args: ''
 tox_posargs: ''
+
+
+# Tobiko test report parameters
+test_report_dir: "{{ tobiko_log_file | dirname }}"
+test_report_prefix: tobiko_results_
+test_report_name: "01_{{ tox_envlist }}"
+collect_files_dir: "{{ zuul.executor.log_root }}"

roles/tobiko/tasks/configure.yaml

@@ -1,31 +0,0 @@
----
-
-- name: "Copy tobiko source code to node: {{ tobiko_source_dir }}/* -> {{ tobiko_dir }}"
-  synchronize:
-    src: "{{ tobiko_source_dir | realpath }}/."
-    dest: "{{ tobiko_dir | realpath }}"
-    use_ssh_args: true
-    recursive: true
-
-
-- become: yes
-  become_user: root
-  block:
-
-    - name: "Provide {{ tobiko_conf_file }} file"
-      ini_file:
-        path: "{{ tobiko_conf_file }}"
-        section: '{{ item.section }}'
-        option: '{{ item.option }}'
-        value: '{{ item.value }}'
-        owner: root
-        mode: '644'
-      loop: "{{ tobiko_conf_options }}"
-
-    - name: "Ensure Tobiko output directories are writable by {{ tobiko_user }} user"
-      file:
-        path: "{{ item | realpath }}"
-        state: directory
-        owner: '{{ tobiko_user }}'
-        mode: '755'
-      loop: "{{ tobiko_output_dirs | unique }}"

roles/tobiko/tasks/main.yaml

@@ -6,30 +6,21 @@
     PYTHON_VERSION: "{{ python_version }}"
   block:
 
-    - name: "Configure Tobiko"
+    - when: tobiko_pre_run | bool
-      when: tobiko_configure | bool
-      include: configure.yaml
-
-    - name: "Check CLI tools"
-      when: tobiko_check_tools | bool
-      include: check_tools.yaml
-
-    - when: tobiko_run_tests
       block:
+        - include: pre-run/source.yaml
+        - include: pre-run/config.yaml
+        - include: pre-run/output.yaml
+        - include: pre-run/tools.yaml
 
-        - name: "Run tests"
+    - when: tobiko_run | bool
+      block:
+        - include: run/tests.yaml
           when: tobiko_run_tests | bool
-          include: tasks/run_tests.yaml
 
-        - when: tobiko_run_faults | bool
+        - include: run/faults.yaml
-          block:
+          when: tobiko_run_faults | bool
 
-            - name: "Run faults"
+    - when: tobiko_post_run | bool
-              include: run_tests.yaml
+      block:
-              vars:
+        - include: post-run/report.yaml
-                tox_envlist: faults
-
-            - name: "Run tests after faults"
-              include: run_tests.yaml
-              vars:
-                tox_posargs: "--combine"

roles/tobiko/tasks/post-run/report.yaml

@@ -0,0 +1,25 @@
+---
+
+- name: "Look for test result files in {{ test_report_dir }}"
+  find:
+    paths:
+      - "{{ test_report_dir }}"
+    patterns:
+      - "{{ test_report_prefix }}*.subunit"
+      - "{{ test_report_prefix }}*.html"
+  register: test_result_files
+
+
+- name: "Compress test result files"
+  archive:
+    path: "{{ item.path }}"
+  with_items: "{{ test_result_files.files }}"
+
+
+- name: "Collect test result files"
+  synchronize:
+    dest: "{{ collect_files_dir }}"
+    mode: pull
+    src: "{{ item.path }}.gz"
+    verify_host: true
+  with_items: "{{ test_result_files.files }}"

roles/tobiko/tasks/pre-run/config.yaml

@@ -0,0 +1,15 @@
+---
+
+- become: yes
+  become_user: root
+  block:
+
+    - name: "Provide {{ tobiko_conf_file }} file"
+      ini_file:
+        path: "{{ tobiko_conf_file }}"
+        section: '{{ item.section }}'
+        option: '{{ item.option }}'
+        value: '{{ item.value }}'
+        owner: root
+        mode: '644'
+      loop: "{{ tobiko_conf_options }}"

roles/tobiko/tasks/pre-run/output.yaml

@@ -0,0 +1,13 @@
+---
+
+- become: yes
+  become_user: root
+  block:
+
+    - name: "Ensure Tobiko output directories are writable by {{ tobiko_user }} user"
+      file:
+        path: "{{ item | realpath }}"
+        state: directory
+        owner: '{{ tobiko_user }}'
+        mode: '755'
+      loop: "{{ tobiko_output_dirs | unique }}"

roles/tobiko/tasks/pre-run/source.yaml

@@ -0,0 +1,15 @@
+---
+
+- name: "Check {{ tobiko_dir }} directory exists"
+  stat:
+    path: "{{ tobiko_dir | realpath }}"
+  register: tobiko_dir_stat
+
+
+- name: "Copy tobiko source code to node: {{ tobiko_source_dir }}/* -> {{ tobiko_dir }}"
+  synchronize:
+    src: "{{ tobiko_source_dir | realpath }}/."
+    dest: "{{ tobiko_dir | realpath }}"
+    use_ssh_args: true
+    recursive: true
+  when: "not tobiko_dir_stat.stat.exists"

roles/tobiko/tasks/run/faults.yaml

@@ -0,0 +1,13 @@
+---
+
+- include: tests.yaml
+  vars:
+    tox_envlist: faults
+    test_report_name: "02_faults"
+
+
+- include: tests.yaml
+  environment:
+    TOBIKO_PREVENT_CREATE=true
+  vars:
+    test_report_name: "03_{{ tox_envlist }}_after_faults"

roles/tobiko/tasks/run/tests.yaml

@@ -0,0 +1,14 @@
+---
+
+- name: "Run test '{{ tox_envlist }}' test cases from '{{ tox_dir }}'"
+  shell:
+    chdir: "{{ tox_dir }}"
+    cmd: |
+        {{ tox_command }} -e '{{ tox_envlist }}' {{ tox_extra_args }} -- \
+            {{ tox_posargs }}
+        exit_status=$?
+        {{ tox_command }} -c '{{ tobiko_dir }}' -e report
+        exit "${exit_status}"
+  environment:
+    TOBIKO_TEST_REPORT_DIR: "{{ test_report_dir }}"
+    TOBIKO_TEST_REPORT_NAME: "{{ test_report_prefix }}{{ test_report_name }}"

roles/tobiko/tasks/run_tests.yaml

@@ -1,8 +0,0 @@
----
-
-- name: "Run test '{{ tox_envlist }}' cases from '{{ tox_dir }}'"
-  shell:
-    chdir: "{{ tox_dir }}"
-    cmd: |
-        {{ tox_command }} -e '{{ tox_envlist }}' {{ tox_extra_args }} -- \
-            {{ tox_posargs }}

tools/ci/make_report

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Produce test HTML report file into ${REPORT_DIR}
+
+set -xeu
+
+# Imput paths
+TOBIKO_TOX_DIR=${TOBIKO_TOX_DIR:-$(pwd)}
+
+# Output dirs
+TOBIKO_TEST_REPORT_DIR=${TOBIKO_TEST_REPORT_DIR:-${TOBIKO_TOX_DIR}}
+
+# Output files
+TOBIKO_TEST_REPORT_NAME=${TOBIKO_TEST_REPORT_NAME:-tobiko_results}
+TOBIKO_TEST_REPORT_SUBUNIT=${TOBIKO_TEST_REPORT_SUBUNIT:-${TOBIKO_TEST_REPORT_DIR}/${TOBIKO_TEST_REPORT_NAME}.subunit}
+TOBIKO_TEST_REPORT_HTML=${TOBIKO_TEST_REPORT_HTML:-${TOBIKO_TEST_REPORT_DIR}/${TOBIKO_TEST_REPORT_NAME}.html}
+TOBIKO_TEST_REPORT_XML=${TOBIKO_TEST_REPORT_XML:-${TOBIKO_TEST_REPORT_DIR}/${TOBIKO_TEST_REPORT_NAME}.xml}
+
+
+function make_report() {
+    make_report_subunit
+    make_report_html
+    make_report_xml
+}
+
+
+function make_report_subunit() {
+    mkdir -p $(dirname "${TOBIKO_TEST_REPORT_SUBUNIT}")
+    (
+        cd "${TOBIKO_TOX_DIR}"
+        stestr last --subunit --force-subunit-trace --all-attachments
+    ) > "${TOBIKO_TEST_REPORT_SUBUNIT}"
+}
+
+
+function make_report_html() {
+    mkdir -p $(dirname "${TOBIKO_TEST_REPORT_HTML}")
+    subunit2html "${TOBIKO_TEST_REPORT_SUBUNIT}" "${TOBIKO_TEST_REPORT_HTML}" > /dev/null
+}
+
+
+function make_report_xml() {
+    mkdir -p $(dirname "${TOBIKO_TEST_REPORT_XML}")
+    subunit2junitxml "${TOBIKO_TEST_REPORT_SUBUNIT}" -o "${TOBIKO_TEST_REPORT_XML}" || true
+}
+
+
+make_report

tox.ini

@@ -12,13 +12,15 @@ deps =
     -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt}
     -r{toxinidir}/requirements.txt
 passenv =
+    TOBIKO_*
 setenv =
-    VIRTUAL_ENV={envdir}
-    PYTHONWARNINGS=ignore::Warning,{env:PYTHONWARNINGS:}
     OS_LOG_CAPTURE={env:OS_LOG_CAPTURE:true}
     OS_STDOUT_CAPTURE={env:OS_STDOUT_CAPTURE:true}
     OS_STDERR_CAPTURE={env:OS_STDERR_CAPTURE:true}
     PYTHON=python
+    PYTHONWARNINGS=ignore::Warning,{env:PYTHONWARNINGS:}
+    TOBIKO_PREVENT_CREATE={env:TOBIKO_PREVENT_CREATE:false}
+    VIRTUAL_ENV={envdir}
 commands =
     stestr run {posargs}
 
@@ -196,5 +198,8 @@ commands =
 # --- CI report environments --------------------------------------------------
 
 [testenv:report]
-deps = -r {toxinidir}/report/requirements.txt
+basepython=python
-commands = {toxinidir}/report/build.sh
+deps = -r {toxinidir}/report-requirements.txt
+passenv = {[base]passenv}
+setenv = {[base]setenv}
+commands = {toxinidir}/tools/ci/make_report

zuul.d/tobiko-devstack.yaml

@@ -62,9 +62,7 @@
               instances: 20
       test_results_stage_name: test_results
       zuul_copy_output:
-        '{{ devstack_log_dir }}/tobiko.log': logs
+        '{{ devstack_log_dir }}/tobiko*.log': logs
-        '{{ stage_dir }}/{{ test_results_stage_name }}.subunit': logs
-        '{{ stage_dir }}/{{ test_results_stage_name }}.html': logs
         '{{ stage_dir }}/stackviz': logs
       extensions_to_txt:
         conf: true
@@ -74,6 +72,7 @@
       tobiko_user: stack
       tobiko_dir: /opt/stack/tobiko
       tobiko_log_file: /opt/stack/logs/tobiko.log
+      test_report_dir: /opt/stack/logs
 
     pre-run: playbooks/tobiko-devstack-pre-run.yaml
     run: playbooks/tobiko-devstack-run.yaml