x/tobiko
Code Issues Proposed changes

Produce separate named report files

Browse Source 
Refactor role with tags for execute pre-run, run and
post-run phases.

Change-Id: I7daff63be56d2746a63e00d21aaa78078e60f60f
This commit is contained in:
Federico Ressi 2019-11-27 17:42:03 +01:00
parent 7c2f566804
commit 88cdecbc4f
21 changed files with 213 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -25,8 +25,7 @@ cover/
doc/build/*
dist/
etc/*.sample
last.subunit
tobiko_results.*
tobiko_results*
zuul/versioninfo
# Files created by releasenotes build

13
playbooks/tobiko-devstack-post-run.yaml
View File

@ -14,9 +14,18 @@
---
- hosts: tempest
roles:
- role: tobiko
tags:
- tobiko-post-run
vars:
tobiko_pre_run: false
tobiko_run: false
tobiko_post_run: true
- hosts: tempest
become: true
roles:
- role: fetch-subunit-output
zuul_work_dir: '{{ devstack_base_dir }}/tobiko'
- role: process-stackviz

22
playbooks/tobiko-devstack-pre-run.yaml
View File

@ -29,18 +29,20 @@
become: yes
- name: Copy the build ssh key to authorized_keys on all hosts for stack
hosts: all
- hosts: all
roles:
- role: copy-build-sshkey
copy_sshkey_target_user: stack
- hosts: all
roles:
- multi-node-setup
- hosts: all
roles:
- orchestrate-devstack
- hosts: tempest
roles:
- role: tobiko
tags:
- tobiko-pre-run
vars:
tobiko_pre_run: true
tobiko_run: false
tobiko_post_run: false

7
playbooks/tobiko-devstack-run.yaml
View File

@ -14,7 +14,10 @@
---
- name: Run Tobiko
hosts: tempest
- hosts: tempest
roles:
- role: tobiko
vars:
tobiko_pre_run: false
tobiko_run: true
tobiko_post_run: false

0
report/requirements.txt → report-requirements.txt
View File

1
report/.gitignore vendored
View File

@ -1 +0,0 @@
/build/

22
report/build.sh
View File

@ -1,22 +0,0 @@
#!/bin/sh
# Produce test HTML report file into ${REPORT_DIR}
set -xeu
TESTR_DIR=${TESTR_DIR:-$(pwd)}
BUILD_DIR=${BUILD_DIR:-${TESTR_DIR}}
SUBUNIT_FILE=${SUBUNIT_FILE:-${BUILD_DIR}/last.subunit}
TESTR_RESULTS_HTML=${TESTR_RESULTS_HTML:-${BUILD_DIR}/tobiko_results.html}
TESTR_RESULTS_XML=${TESTR_RESULTS_XML:-${BUILD_DIR}/tobiko_results.xml}
make_testr_results_html() {
mkdir -p "$(dirname ${SUBUNIT_FILE})"
(cd "${TESTR_DIR}" && stestr last --subunit) > "${SUBUNIT_FILE}"
(cd "${BUILD_DIR}" && subunit2html "${SUBUNIT_FILE}" "${TESTR_RESULTS_HTML}")
(cd "${BUILD_DIR}" && subunit2junitxml "${SUBUNIT_FILE}" -o "${TESTR_RESULTS_XML}")
}
make_testr_results_html

24
roles/tobiko/defaults/main.yaml
View File

@ -1,10 +1,11 @@
---
# Tobiko workflow parameters
tobiko_configure: true
tobiko_check_tools: true
# Tobiko workflow flags
tobiko_pre_run: true
tobiko_run: "{{ tobiko_run_tests or tobiko_run_faults }}"
tobiko_run_tests: true
tobiko_run_faults: false
tobiko_post_run: true
# Tobiko sources tobiko directory
@ -12,10 +13,6 @@ tobiko_source_dir: "{{ role_path | dirname | dirname }}"
tobiko_dir: "{{ ansible_env.HOME }}/src/tobiko"
tobiko_user: "{{ ansible_user }}"
# Logging options
tobiko_debug: True
tobiko_log_file: "{{ tobiko_dir }}/tobiko.log"
# Configuration options
tobiko_conf_file: /etc/tobiko/tobiko.conf
@ -34,10 +31,16 @@ tobiko_conf_options:
value: "{{ tobiko_log_file | basename }}"
# Logging options
tobiko_debug: True
tobiko_log_file: "{{ tobiko_dir }}/tobiko.log"
# Output directories where tobiko should write to
tobiko_output_dirs:
- "{{ tox_dir }}"
- "{{ tobiko_log_file | dirname }}"
- "{{ test_report_dir }}"
tobiko_become_user: ''
@ -54,3 +57,10 @@ tox_command: "{{ tobiko_dir }}/tools/ci/tox"
tox_envlist: scenario
tox_extra_args: ''
tox_posargs: ''
# Tobiko test report parameters
test_report_dir: "{{ tobiko_log_file | dirname }}"
test_report_prefix: tobiko_results_
test_report_name: "01_{{ tox_envlist }}"
collect_files_dir: "{{ zuul.executor.log_root }}"

31
roles/tobiko/tasks/configure.yaml
View File

@ -1,31 +0,0 @@
---
- name: "Copy tobiko source code to node: {{ tobiko_source_dir }}/* -> {{ tobiko_dir }}"
synchronize:
src: "{{ tobiko_source_dir | realpath }}/."
dest: "{{ tobiko_dir | realpath }}"
use_ssh_args: true
recursive: true
- become: yes
become_user: root
block:
- name: "Provide {{ tobiko_conf_file }} file"
ini_file:
path: "{{ tobiko_conf_file }}"
section: '{{ item.section }}'
option: '{{ item.option }}'
value: '{{ item.value }}'
owner: root
mode: '644'
loop: "{{ tobiko_conf_options }}"
- name: "Ensure Tobiko output directories are writable by {{ tobiko_user }} user"
file:
path: "{{ item | realpath }}"
state: directory
owner: '{{ tobiko_user }}'
mode: '755'
loop: "{{ tobiko_output_dirs | unique }}"

35
roles/tobiko/tasks/main.yaml
View File

@ -6,30 +6,21 @@
PYTHON_VERSION: "{{ python_version }}"
block:
- name: "Configure Tobiko"
when: tobiko_configure | bool
include: configure.yaml
- name: "Check CLI tools"
when: tobiko_check_tools | bool
include: check_tools.yaml
- when: tobiko_run_tests
- when: tobiko_pre_run | bool
block:
- include: pre-run/source.yaml
- include: pre-run/config.yaml
- include: pre-run/output.yaml
- include: pre-run/tools.yaml
- name: "Run tests"
- when: tobiko_run | bool
block:
- include: run/tests.yaml
when: tobiko_run_tests | bool
include: tasks/run_tests.yaml
- when: tobiko_run_faults | bool
block:
- include: run/faults.yaml
when: tobiko_run_faults | bool
- name: "Run faults"
include: run_tests.yaml
vars:
tox_envlist: faults
- name: "Run tests after faults"
include: run_tests.yaml
vars:
tox_posargs: "--combine"
- when: tobiko_post_run | bool
block:
- include: post-run/report.yaml

25
roles/tobiko/tasks/post-run/report.yaml Normal file
View File

@ -0,0 +1,25 @@
---
- name: "Look for test result files in {{ test_report_dir }}"
find:
paths:
- "{{ test_report_dir }}"
patterns:
- "{{ test_report_prefix }}*.subunit"
- "{{ test_report_prefix }}*.html"
register: test_result_files
- name: "Compress test result files"
archive:
path: "{{ item.path }}"
with_items: "{{ test_result_files.files }}"
- name: "Collect test result files"
synchronize:
dest: "{{ collect_files_dir }}"
mode: pull
src: "{{ item.path }}.gz"
verify_host: true
with_items: "{{ test_result_files.files }}"

15
roles/tobiko/tasks/pre-run/config.yaml Normal file
View File

@ -0,0 +1,15 @@
---
- become: yes
become_user: root
block:
- name: "Provide {{ tobiko_conf_file }} file"
ini_file:
path: "{{ tobiko_conf_file }}"
section: '{{ item.section }}'
option: '{{ item.option }}'
value: '{{ item.value }}'
owner: root
mode: '644'
loop: "{{ tobiko_conf_options }}"

13
roles/tobiko/tasks/pre-run/output.yaml Normal file
View File

@ -0,0 +1,13 @@
---
- become: yes
become_user: root
block:
- name: "Ensure Tobiko output directories are writable by {{ tobiko_user }} user"
file:
path: "{{ item | realpath }}"
state: directory
owner: '{{ tobiko_user }}'
mode: '755'
loop: "{{ tobiko_output_dirs | unique }}"

15
roles/tobiko/tasks/pre-run/source.yaml Normal file
View File

@ -0,0 +1,15 @@
---
- name: "Check {{ tobiko_dir }} directory exists"
stat:
path: "{{ tobiko_dir | realpath }}"
register: tobiko_dir_stat
- name: "Copy tobiko source code to node: {{ tobiko_source_dir }}/* -> {{ tobiko_dir }}"
synchronize:
src: "{{ tobiko_source_dir | realpath }}/."
dest: "{{ tobiko_dir | realpath }}"
use_ssh_args: true
recursive: true
when: "not tobiko_dir_stat.stat.exists"

0
roles/tobiko/tasks/check_tools.yaml → roles/tobiko/tasks/pre-run/tools.yaml
View File

13
roles/tobiko/tasks/run/faults.yaml Normal file
View File

@ -0,0 +1,13 @@
---
- include: tests.yaml
vars:
tox_envlist: faults
test_report_name: "02_faults"
- include: tests.yaml
environment:
TOBIKO_PREVENT_CREATE=true
vars:
test_report_name: "03_{{ tox_envlist }}_after_faults"

14
roles/tobiko/tasks/run/tests.yaml Normal file
View File

@ -0,0 +1,14 @@
---
- name: "Run test '{{ tox_envlist }}' test cases from '{{ tox_dir }}'"
shell:
chdir: "{{ tox_dir }}"
cmd: |
{{ tox_command }} -e '{{ tox_envlist }}' {{ tox_extra_args }} -- \
{{ tox_posargs }}
exit_status=$?
{{ tox_command }} -c '{{ tobiko_dir }}' -e report
exit "${exit_status}"
environment:
TOBIKO_TEST_REPORT_DIR: "{{ test_report_dir }}"
TOBIKO_TEST_REPORT_NAME: "{{ test_report_prefix }}{{ test_report_name }}"

8
roles/tobiko/tasks/run_tests.yaml
View File

@ -1,8 +0,0 @@
---
- name: "Run test '{{ tox_envlist }}' cases from '{{ tox_dir }}'"
shell:
chdir: "{{ tox_dir }}"
cmd: |
{{ tox_command }} -e '{{ tox_envlist }}' {{ tox_extra_args }} -- \
{{ tox_posargs }}

48
tools/ci/make_report Executable file
View File

@ -0,0 +1,48 @@
#!/bin/bash
# Produce test HTML report file into ${REPORT_DIR}
set -xeu
# Imput paths
TOBIKO_TOX_DIR=${TOBIKO_TOX_DIR:-$(pwd)}
# Output dirs
TOBIKO_TEST_REPORT_DIR=${TOBIKO_TEST_REPORT_DIR:-${TOBIKO_TOX_DIR}}
# Output files
TOBIKO_TEST_REPORT_NAME=${TOBIKO_TEST_REPORT_NAME:-tobiko_results}
TOBIKO_TEST_REPORT_SUBUNIT=${TOBIKO_TEST_REPORT_SUBUNIT:-${TOBIKO_TEST_REPORT_DIR}/${TOBIKO_TEST_REPORT_NAME}.subunit}
TOBIKO_TEST_REPORT_HTML=${TOBIKO_TEST_REPORT_HTML:-${TOBIKO_TEST_REPORT_DIR}/${TOBIKO_TEST_REPORT_NAME}.html}
TOBIKO_TEST_REPORT_XML=${TOBIKO_TEST_REPORT_XML:-${TOBIKO_TEST_REPORT_DIR}/${TOBIKO_TEST_REPORT_NAME}.xml}
function make_report() {
make_report_subunit
make_report_html
make_report_xml
}
function make_report_subunit() {
mkdir -p $(dirname "${TOBIKO_TEST_REPORT_SUBUNIT}")
(
cd "${TOBIKO_TOX_DIR}"
stestr last --subunit --force-subunit-trace --all-attachments
) > "${TOBIKO_TEST_REPORT_SUBUNIT}"
}
function make_report_html() {
mkdir -p $(dirname "${TOBIKO_TEST_REPORT_HTML}")
subunit2html "${TOBIKO_TEST_REPORT_SUBUNIT}" "${TOBIKO_TEST_REPORT_HTML}" > /dev/null
}
function make_report_xml() {
mkdir -p $(dirname "${TOBIKO_TEST_REPORT_XML}")
subunit2junitxml "${TOBIKO_TEST_REPORT_SUBUNIT}" -o "${TOBIKO_TEST_REPORT_XML}" || true
}
make_report

13
tox.ini
View File

@ -12,13 +12,15 @@ deps =
-c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt}
-r{toxinidir}/requirements.txt
passenv =
TOBIKO_*
setenv =
VIRTUAL_ENV={envdir}
PYTHONWARNINGS=ignore::Warning,{env:PYTHONWARNINGS:}
OS_LOG_CAPTURE={env:OS_LOG_CAPTURE:true}
OS_STDOUT_CAPTURE={env:OS_STDOUT_CAPTURE:true}
OS_STDERR_CAPTURE={env:OS_STDERR_CAPTURE:true}
PYTHON=python
PYTHONWARNINGS=ignore::Warning,{env:PYTHONWARNINGS:}
TOBIKO_PREVENT_CREATE={env:TOBIKO_PREVENT_CREATE:false}
VIRTUAL_ENV={envdir}
commands =
stestr run {posargs}
@ -196,5 +198,8 @@ commands =
# --- CI report environments --------------------------------------------------
[testenv:report]
deps = -r {toxinidir}/report/requirements.txt
commands = {toxinidir}/report/build.sh
basepython=python
deps = -r {toxinidir}/report-requirements.txt
passenv = {[base]passenv}
setenv = {[base]setenv}
commands = {toxinidir}/tools/ci/make_report

5
zuul.d/tobiko-devstack.yaml
View File

@ -62,9 +62,7 @@
instances: 20
test_results_stage_name: test_results
zuul_copy_output:
'{{ devstack_log_dir }}/tobiko.log': logs
'{{ stage_dir }}/{{ test_results_stage_name }}.subunit': logs
'{{ stage_dir }}/{{ test_results_stage_name }}.html': logs
'{{ devstack_log_dir }}/tobiko*.log': logs
'{{ stage_dir }}/stackviz': logs
extensions_to_txt:
conf: true
@ -74,6 +72,7 @@
tobiko_user: stack
tobiko_dir: /opt/stack/tobiko
tobiko_log_file: /opt/stack/logs/tobiko.log
test_report_dir: /opt/stack/logs
pre-run: playbooks/tobiko-devstack-pre-run.yaml
run: playbooks/tobiko-devstack-run.yaml