From 88cdecbc4fc5ab9d71a50542f51b4728876ec9ba Mon Sep 17 00:00:00 2001 From: Federico Ressi Date: Wed, 27 Nov 2019 17:42:03 +0100 Subject: [PATCH] Produce separate named report files Refactor role with tags for execute pre-run, run and post-run phases. Change-Id: I7daff63be56d2746a63e00d21aaa78078e60f60f --- .gitignore | 3 +- playbooks/tobiko-devstack-post-run.yaml | 13 ++++- playbooks/tobiko-devstack-pre-run.yaml | 22 +++++---- playbooks/tobiko-devstack-run.yaml | 7 ++- ...equirements.txt => report-requirements.txt | 0 report/.gitignore | 1 - report/build.sh | 22 --------- roles/tobiko/defaults/main.yaml | 24 +++++++--- roles/tobiko/tasks/configure.yaml | 31 ------------ roles/tobiko/tasks/main.yaml | 35 +++++--------- roles/tobiko/tasks/post-run/report.yaml | 25 ++++++++++ roles/tobiko/tasks/pre-run/config.yaml | 15 ++++++ roles/tobiko/tasks/pre-run/output.yaml | 13 +++++ roles/tobiko/tasks/pre-run/source.yaml | 15 ++++++ .../{check_tools.yaml => pre-run/tools.yaml} | 0 roles/tobiko/tasks/run/faults.yaml | 13 +++++ roles/tobiko/tasks/run/tests.yaml | 14 ++++++ roles/tobiko/tasks/run_tests.yaml | 8 ---- tools/ci/make_report | 48 +++++++++++++++++++ tox.ini | 13 +++-- zuul.d/tobiko-devstack.yaml | 5 +- 21 files changed, 213 insertions(+), 114 deletions(-) rename report/requirements.txt => report-requirements.txt (100%) delete mode 100644 report/.gitignore delete mode 100755 report/build.sh delete mode 100644 roles/tobiko/tasks/configure.yaml create mode 100644 roles/tobiko/tasks/post-run/report.yaml create mode 100644 roles/tobiko/tasks/pre-run/config.yaml create mode 100644 roles/tobiko/tasks/pre-run/output.yaml create mode 100644 roles/tobiko/tasks/pre-run/source.yaml rename roles/tobiko/tasks/{check_tools.yaml => pre-run/tools.yaml} (100%) create mode 100644 roles/tobiko/tasks/run/faults.yaml create mode 100644 roles/tobiko/tasks/run/tests.yaml delete mode 100644 roles/tobiko/tasks/run_tests.yaml create mode 100755 tools/ci/make_report diff --git a/.gitignore b/.gitignore index d988d2b47..405155413 100644 --- a/.gitignore +++ b/.gitignore @@ -25,8 +25,7 @@ cover/ doc/build/* dist/ etc/*.sample -last.subunit -tobiko_results.* +tobiko_results* zuul/versioninfo # Files created by releasenotes build diff --git a/playbooks/tobiko-devstack-post-run.yaml b/playbooks/tobiko-devstack-post-run.yaml index 5633b648a..54e883bc5 100644 --- a/playbooks/tobiko-devstack-post-run.yaml +++ b/playbooks/tobiko-devstack-post-run.yaml @@ -14,9 +14,18 @@ --- +- hosts: tempest + roles: + - role: tobiko + tags: + - tobiko-post-run + vars: + tobiko_pre_run: false + tobiko_run: false + tobiko_post_run: true + + - hosts: tempest become: true roles: - - role: fetch-subunit-output - zuul_work_dir: '{{ devstack_base_dir }}/tobiko' - role: process-stackviz diff --git a/playbooks/tobiko-devstack-pre-run.yaml b/playbooks/tobiko-devstack-pre-run.yaml index 1bb44f572..e2b2fbcb9 100644 --- a/playbooks/tobiko-devstack-pre-run.yaml +++ b/playbooks/tobiko-devstack-pre-run.yaml @@ -29,18 +29,20 @@ become: yes -- name: Copy the build ssh key to authorized_keys on all hosts for stack - hosts: all +- hosts: all roles: - role: copy-build-sshkey copy_sshkey_target_user: stack - - -- hosts: all - roles: - multi-node-setup - - -- hosts: all - roles: - orchestrate-devstack + + +- hosts: tempest + roles: + - role: tobiko + tags: + - tobiko-pre-run + vars: + tobiko_pre_run: true + tobiko_run: false + tobiko_post_run: false diff --git a/playbooks/tobiko-devstack-run.yaml b/playbooks/tobiko-devstack-run.yaml index c62b57720..3f2a8e5ce 100644 --- a/playbooks/tobiko-devstack-run.yaml +++ b/playbooks/tobiko-devstack-run.yaml @@ -14,7 +14,10 @@ --- -- name: Run Tobiko - hosts: tempest +- hosts: tempest roles: - role: tobiko + vars: + tobiko_pre_run: false + tobiko_run: true + tobiko_post_run: false diff --git a/report/requirements.txt b/report-requirements.txt similarity index 100% rename from report/requirements.txt rename to report-requirements.txt diff --git a/report/.gitignore b/report/.gitignore deleted file mode 100644 index 84c048a73..000000000 --- a/report/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/build/ diff --git a/report/build.sh b/report/build.sh deleted file mode 100755 index 27d29fb9f..000000000 --- a/report/build.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -# Produce test HTML report file into ${REPORT_DIR} - -set -xeu - - -TESTR_DIR=${TESTR_DIR:-$(pwd)} -BUILD_DIR=${BUILD_DIR:-${TESTR_DIR}} -SUBUNIT_FILE=${SUBUNIT_FILE:-${BUILD_DIR}/last.subunit} -TESTR_RESULTS_HTML=${TESTR_RESULTS_HTML:-${BUILD_DIR}/tobiko_results.html} -TESTR_RESULTS_XML=${TESTR_RESULTS_XML:-${BUILD_DIR}/tobiko_results.xml} - - -make_testr_results_html() { - mkdir -p "$(dirname ${SUBUNIT_FILE})" - (cd "${TESTR_DIR}" && stestr last --subunit) > "${SUBUNIT_FILE}" - (cd "${BUILD_DIR}" && subunit2html "${SUBUNIT_FILE}" "${TESTR_RESULTS_HTML}") - (cd "${BUILD_DIR}" && subunit2junitxml "${SUBUNIT_FILE}" -o "${TESTR_RESULTS_XML}") -} - -make_testr_results_html diff --git a/roles/tobiko/defaults/main.yaml b/roles/tobiko/defaults/main.yaml index 0fe18f0ce..bc1f79634 100644 --- a/roles/tobiko/defaults/main.yaml +++ b/roles/tobiko/defaults/main.yaml @@ -1,10 +1,11 @@ --- -# Tobiko workflow parameters -tobiko_configure: true -tobiko_check_tools: true +# Tobiko workflow flags +tobiko_pre_run: true +tobiko_run: "{{ tobiko_run_tests or tobiko_run_faults }}" tobiko_run_tests: true tobiko_run_faults: false +tobiko_post_run: true # Tobiko sources tobiko directory @@ -12,10 +13,6 @@ tobiko_source_dir: "{{ role_path | dirname | dirname }}" tobiko_dir: "{{ ansible_env.HOME }}/src/tobiko" tobiko_user: "{{ ansible_user }}" -# Logging options -tobiko_debug: True -tobiko_log_file: "{{ tobiko_dir }}/tobiko.log" - # Configuration options tobiko_conf_file: /etc/tobiko/tobiko.conf @@ -34,10 +31,16 @@ tobiko_conf_options: value: "{{ tobiko_log_file | basename }}" +# Logging options +tobiko_debug: True +tobiko_log_file: "{{ tobiko_dir }}/tobiko.log" + + # Output directories where tobiko should write to tobiko_output_dirs: - "{{ tox_dir }}" - "{{ tobiko_log_file | dirname }}" + - "{{ test_report_dir }}" tobiko_become_user: '' @@ -54,3 +57,10 @@ tox_command: "{{ tobiko_dir }}/tools/ci/tox" tox_envlist: scenario tox_extra_args: '' tox_posargs: '' + + +# Tobiko test report parameters +test_report_dir: "{{ tobiko_log_file | dirname }}" +test_report_prefix: tobiko_results_ +test_report_name: "01_{{ tox_envlist }}" +collect_files_dir: "{{ zuul.executor.log_root }}" diff --git a/roles/tobiko/tasks/configure.yaml b/roles/tobiko/tasks/configure.yaml deleted file mode 100644 index b60cdc62f..000000000 --- a/roles/tobiko/tasks/configure.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: "Copy tobiko source code to node: {{ tobiko_source_dir }}/* -> {{ tobiko_dir }}" - synchronize: - src: "{{ tobiko_source_dir | realpath }}/." - dest: "{{ tobiko_dir | realpath }}" - use_ssh_args: true - recursive: true - - -- become: yes - become_user: root - block: - - - name: "Provide {{ tobiko_conf_file }} file" - ini_file: - path: "{{ tobiko_conf_file }}" - section: '{{ item.section }}' - option: '{{ item.option }}' - value: '{{ item.value }}' - owner: root - mode: '644' - loop: "{{ tobiko_conf_options }}" - - - name: "Ensure Tobiko output directories are writable by {{ tobiko_user }} user" - file: - path: "{{ item | realpath }}" - state: directory - owner: '{{ tobiko_user }}' - mode: '755' - loop: "{{ tobiko_output_dirs | unique }}" diff --git a/roles/tobiko/tasks/main.yaml b/roles/tobiko/tasks/main.yaml index a24b5e81b..e0dd44bab 100644 --- a/roles/tobiko/tasks/main.yaml +++ b/roles/tobiko/tasks/main.yaml @@ -6,30 +6,21 @@ PYTHON_VERSION: "{{ python_version }}" block: - - name: "Configure Tobiko" - when: tobiko_configure | bool - include: configure.yaml - - - name: "Check CLI tools" - when: tobiko_check_tools | bool - include: check_tools.yaml - - - when: tobiko_run_tests + - when: tobiko_pre_run | bool block: + - include: pre-run/source.yaml + - include: pre-run/config.yaml + - include: pre-run/output.yaml + - include: pre-run/tools.yaml - - name: "Run tests" + - when: tobiko_run | bool + block: + - include: run/tests.yaml when: tobiko_run_tests | bool - include: tasks/run_tests.yaml - - when: tobiko_run_faults | bool - block: + - include: run/faults.yaml + when: tobiko_run_faults | bool - - name: "Run faults" - include: run_tests.yaml - vars: - tox_envlist: faults - - - name: "Run tests after faults" - include: run_tests.yaml - vars: - tox_posargs: "--combine" + - when: tobiko_post_run | bool + block: + - include: post-run/report.yaml diff --git a/roles/tobiko/tasks/post-run/report.yaml b/roles/tobiko/tasks/post-run/report.yaml new file mode 100644 index 000000000..83aeaa138 --- /dev/null +++ b/roles/tobiko/tasks/post-run/report.yaml @@ -0,0 +1,25 @@ +--- + +- name: "Look for test result files in {{ test_report_dir }}" + find: + paths: + - "{{ test_report_dir }}" + patterns: + - "{{ test_report_prefix }}*.subunit" + - "{{ test_report_prefix }}*.html" + register: test_result_files + + +- name: "Compress test result files" + archive: + path: "{{ item.path }}" + with_items: "{{ test_result_files.files }}" + + +- name: "Collect test result files" + synchronize: + dest: "{{ collect_files_dir }}" + mode: pull + src: "{{ item.path }}.gz" + verify_host: true + with_items: "{{ test_result_files.files }}" diff --git a/roles/tobiko/tasks/pre-run/config.yaml b/roles/tobiko/tasks/pre-run/config.yaml new file mode 100644 index 000000000..c36ec93df --- /dev/null +++ b/roles/tobiko/tasks/pre-run/config.yaml @@ -0,0 +1,15 @@ +--- + +- become: yes + become_user: root + block: + + - name: "Provide {{ tobiko_conf_file }} file" + ini_file: + path: "{{ tobiko_conf_file }}" + section: '{{ item.section }}' + option: '{{ item.option }}' + value: '{{ item.value }}' + owner: root + mode: '644' + loop: "{{ tobiko_conf_options }}" diff --git a/roles/tobiko/tasks/pre-run/output.yaml b/roles/tobiko/tasks/pre-run/output.yaml new file mode 100644 index 000000000..c1dd02924 --- /dev/null +++ b/roles/tobiko/tasks/pre-run/output.yaml @@ -0,0 +1,13 @@ +--- + +- become: yes + become_user: root + block: + + - name: "Ensure Tobiko output directories are writable by {{ tobiko_user }} user" + file: + path: "{{ item | realpath }}" + state: directory + owner: '{{ tobiko_user }}' + mode: '755' + loop: "{{ tobiko_output_dirs | unique }}" diff --git a/roles/tobiko/tasks/pre-run/source.yaml b/roles/tobiko/tasks/pre-run/source.yaml new file mode 100644 index 000000000..d15dd520b --- /dev/null +++ b/roles/tobiko/tasks/pre-run/source.yaml @@ -0,0 +1,15 @@ +--- + +- name: "Check {{ tobiko_dir }} directory exists" + stat: + path: "{{ tobiko_dir | realpath }}" + register: tobiko_dir_stat + + +- name: "Copy tobiko source code to node: {{ tobiko_source_dir }}/* -> {{ tobiko_dir }}" + synchronize: + src: "{{ tobiko_source_dir | realpath }}/." + dest: "{{ tobiko_dir | realpath }}" + use_ssh_args: true + recursive: true + when: "not tobiko_dir_stat.stat.exists" diff --git a/roles/tobiko/tasks/check_tools.yaml b/roles/tobiko/tasks/pre-run/tools.yaml similarity index 100% rename from roles/tobiko/tasks/check_tools.yaml rename to roles/tobiko/tasks/pre-run/tools.yaml diff --git a/roles/tobiko/tasks/run/faults.yaml b/roles/tobiko/tasks/run/faults.yaml new file mode 100644 index 000000000..7c262d53b --- /dev/null +++ b/roles/tobiko/tasks/run/faults.yaml @@ -0,0 +1,13 @@ +--- + +- include: tests.yaml + vars: + tox_envlist: faults + test_report_name: "02_faults" + + +- include: tests.yaml + environment: + TOBIKO_PREVENT_CREATE=true + vars: + test_report_name: "03_{{ tox_envlist }}_after_faults" diff --git a/roles/tobiko/tasks/run/tests.yaml b/roles/tobiko/tasks/run/tests.yaml new file mode 100644 index 000000000..aad62a669 --- /dev/null +++ b/roles/tobiko/tasks/run/tests.yaml @@ -0,0 +1,14 @@ +--- + +- name: "Run test '{{ tox_envlist }}' test cases from '{{ tox_dir }}'" + shell: + chdir: "{{ tox_dir }}" + cmd: | + {{ tox_command }} -e '{{ tox_envlist }}' {{ tox_extra_args }} -- \ + {{ tox_posargs }} + exit_status=$? + {{ tox_command }} -c '{{ tobiko_dir }}' -e report + exit "${exit_status}" + environment: + TOBIKO_TEST_REPORT_DIR: "{{ test_report_dir }}" + TOBIKO_TEST_REPORT_NAME: "{{ test_report_prefix }}{{ test_report_name }}" diff --git a/roles/tobiko/tasks/run_tests.yaml b/roles/tobiko/tasks/run_tests.yaml deleted file mode 100644 index e03c107bf..000000000 --- a/roles/tobiko/tasks/run_tests.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- - -- name: "Run test '{{ tox_envlist }}' cases from '{{ tox_dir }}'" - shell: - chdir: "{{ tox_dir }}" - cmd: | - {{ tox_command }} -e '{{ tox_envlist }}' {{ tox_extra_args }} -- \ - {{ tox_posargs }} diff --git a/tools/ci/make_report b/tools/ci/make_report new file mode 100755 index 000000000..c100848f0 --- /dev/null +++ b/tools/ci/make_report @@ -0,0 +1,48 @@ +#!/bin/bash + +# Produce test HTML report file into ${REPORT_DIR} + +set -xeu + +# Imput paths +TOBIKO_TOX_DIR=${TOBIKO_TOX_DIR:-$(pwd)} + +# Output dirs +TOBIKO_TEST_REPORT_DIR=${TOBIKO_TEST_REPORT_DIR:-${TOBIKO_TOX_DIR}} + +# Output files +TOBIKO_TEST_REPORT_NAME=${TOBIKO_TEST_REPORT_NAME:-tobiko_results} +TOBIKO_TEST_REPORT_SUBUNIT=${TOBIKO_TEST_REPORT_SUBUNIT:-${TOBIKO_TEST_REPORT_DIR}/${TOBIKO_TEST_REPORT_NAME}.subunit} +TOBIKO_TEST_REPORT_HTML=${TOBIKO_TEST_REPORT_HTML:-${TOBIKO_TEST_REPORT_DIR}/${TOBIKO_TEST_REPORT_NAME}.html} +TOBIKO_TEST_REPORT_XML=${TOBIKO_TEST_REPORT_XML:-${TOBIKO_TEST_REPORT_DIR}/${TOBIKO_TEST_REPORT_NAME}.xml} + + +function make_report() { + make_report_subunit + make_report_html + make_report_xml +} + + +function make_report_subunit() { + mkdir -p $(dirname "${TOBIKO_TEST_REPORT_SUBUNIT}") + ( + cd "${TOBIKO_TOX_DIR}" + stestr last --subunit --force-subunit-trace --all-attachments + ) > "${TOBIKO_TEST_REPORT_SUBUNIT}" +} + + +function make_report_html() { + mkdir -p $(dirname "${TOBIKO_TEST_REPORT_HTML}") + subunit2html "${TOBIKO_TEST_REPORT_SUBUNIT}" "${TOBIKO_TEST_REPORT_HTML}" > /dev/null +} + + +function make_report_xml() { + mkdir -p $(dirname "${TOBIKO_TEST_REPORT_XML}") + subunit2junitxml "${TOBIKO_TEST_REPORT_SUBUNIT}" -o "${TOBIKO_TEST_REPORT_XML}" || true +} + + +make_report diff --git a/tox.ini b/tox.ini index 9b4fc7dd2..54e335dc0 100644 --- a/tox.ini +++ b/tox.ini @@ -12,13 +12,15 @@ deps = -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt} -r{toxinidir}/requirements.txt passenv = + TOBIKO_* setenv = - VIRTUAL_ENV={envdir} - PYTHONWARNINGS=ignore::Warning,{env:PYTHONWARNINGS:} OS_LOG_CAPTURE={env:OS_LOG_CAPTURE:true} OS_STDOUT_CAPTURE={env:OS_STDOUT_CAPTURE:true} OS_STDERR_CAPTURE={env:OS_STDERR_CAPTURE:true} PYTHON=python + PYTHONWARNINGS=ignore::Warning,{env:PYTHONWARNINGS:} + TOBIKO_PREVENT_CREATE={env:TOBIKO_PREVENT_CREATE:false} + VIRTUAL_ENV={envdir} commands = stestr run {posargs} @@ -196,5 +198,8 @@ commands = # --- CI report environments -------------------------------------------------- [testenv:report] -deps = -r {toxinidir}/report/requirements.txt -commands = {toxinidir}/report/build.sh +basepython=python +deps = -r {toxinidir}/report-requirements.txt +passenv = {[base]passenv} +setenv = {[base]setenv} +commands = {toxinidir}/tools/ci/make_report diff --git a/zuul.d/tobiko-devstack.yaml b/zuul.d/tobiko-devstack.yaml index f0eed2185..8a8c16b2b 100644 --- a/zuul.d/tobiko-devstack.yaml +++ b/zuul.d/tobiko-devstack.yaml @@ -62,9 +62,7 @@ instances: 20 test_results_stage_name: test_results zuul_copy_output: - '{{ devstack_log_dir }}/tobiko.log': logs - '{{ stage_dir }}/{{ test_results_stage_name }}.subunit': logs - '{{ stage_dir }}/{{ test_results_stage_name }}.html': logs + '{{ devstack_log_dir }}/tobiko*.log': logs '{{ stage_dir }}/stackviz': logs extensions_to_txt: conf: true @@ -74,6 +72,7 @@ tobiko_user: stack tobiko_dir: /opt/stack/tobiko tobiko_log_file: /opt/stack/logs/tobiko.log + test_report_dir: /opt/stack/logs pre-run: playbooks/tobiko-devstack-pre-run.yaml run: playbooks/tobiko-devstack-run.yaml