fetch-subunit-output: collect additional subunits (2nd try)

In addition to the main subunit file from zuul_work_dir,
collect the subunit files from the elements (directories)
of the zuul_additional_subunit_dirs list.

The default behavior is unchanged.

While the documentation of this role states that zuul_work_dir
contains an absolute path, this is not always true.
So make sure to not make any assumption about zuul_work_dir
in order to not fail spectacularly as it happened with the
previous patch[0].

Add also some tests for the role: both the basic case
and with an additional test directory.

[0] https://review.opendev.org/673885

Change-Id: Iabf2e0cf6d86e36a174778367186bbd39a65c3dd
This commit is contained in:
Luigi Toscano 2019-10-17 00:42:13 +02:00
parent 6d5826844a
commit ab8f9fc403
9 changed files with 207 additions and 2 deletions

View File

@ -7,6 +7,13 @@ Collect subunit outputs
Directory to work in. It has to be a fully qualified path. Directory to work in. It has to be a fully qualified path.
.. zuul:rolevar:: zuul_additional_subunit_dirs
:default: []
List of additional directories which contains subunit files
to collect. The content of zuul_work_dir is always checked,
so it should not be added here.
.. zuul:rolevar:: tox_envlist .. zuul:rolevar:: tox_envlist
tox environment that was used to run the tests originally. tox environment that was used to run the tests originally.

View File

@ -1,3 +1,4 @@
--- ---
tox_envlist: "" tox_envlist: ""
zuul_additional_subunit_dirs: []
zuul_work_dir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}" zuul_work_dir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"

View File

@ -9,11 +9,37 @@
- testr_command.rc == 0 - testr_command.rc == 0
- testr_command.stdout_lines - testr_command.stdout_lines
block: block:
- name: Get the list of directories with subunit files
set_fact:
all_subunit_dirs: "{{ [ zuul_work_dir ] + zuul_additional_subunit_dirs }}"
# The usage an independent target file instead of sending the output
# to zuul_work_dir prevents issues related to zuul_work_dir being
# a relative path, which may happen despite what the documentation
# of this role claims.
- name: Create a temporary file to store the subunit stream
tempfile:
state: file
prefix: subunit.
register: temp_subunit_file
- name: Generate subunit file - name: Generate subunit file
shell: shell:
cmd: "{{ testr_command.stdout_lines[0] }} last --subunit > ./testrepository.subunit" cmd: "{{ testr_command.stdout_lines[0] }} last --subunit >>{{ temp_subunit_file.path }}"
chdir: "{{ zuul_work_dir }}" chdir: "{{ item }}"
loop: "{{ all_subunit_dirs }}"
- name: Copy the combined subunit file to the zuul work directory
copy:
src: "{{ temp_subunit_file.path }}"
dest: "{{ zuul_work_dir }}/testrepository.subunit"
remote_src: yes
- name: Remove the temporary file
file:
name: "{{ temp_subunit_file.path }}"
state: absent
ignore_errors: true
- name: Process and fetch subunit results - name: Process and fetch subunit results
include: process.yaml include: process.yaml

View File

@ -11,3 +11,4 @@
- include: fetch-zuul-cloner.yaml - include: fetch-zuul-cloner.yaml
- include: validate-host.yaml - include: validate-host.yaml
- include: fetch-output.yaml - include: fetch-output.yaml
- include: fetch-subunit-output.yaml

View File

@ -0,0 +1,127 @@
- name: Run the fetch-subunit-output role
hosts: all
vars:
tests_data:
main:
directory: "{{ zuul_work_dir }}"
test_pattern: "WorkingTest.test_success"
secondary:
directory: "/var/tmp/extratests"
test_pattern: "FailingTest.test_failure"
pre_tasks:
# Required packages; install them into a .tox path
# to cover the find-*.sh scripts in the role a bit more.
- name: Install stestr and subunit-output
pip:
name:
- stestr>=2.0.0,<2.6.0
- python-subunit
virtualenv: "{{ zuul_work_dir }}/.tox/utests/"
- name: Ensure that the test directories exists
file:
name: "{{ item.value.directory }}"
state: directory
loop: "{{ tests_data|dict2items }}"
- name: Copy the test files on all directories
copy:
src: "subunit_tests"
dest: "{{ item.value.directory }}"
loop: "{{ tests_data|dict2items }}"
- name: Prepare the test results on all directories
shell: |
. {{ zuul_work_dir }}/.tox/utests/bin/activate
stestr init
stestr run --test-path subunit_tests {{ item.value.test_pattern }}
args:
chdir: "{{ item.value.directory }}"
ignore_errors: yes
loop: "{{ tests_data|dict2items }}"
roles:
- role: fetch-subunit-output
post_tasks:
- name: Check that the testrepository file has been pulled
delegate_to: localhost
file:
path: "{{ zuul.executor.log_root }}/testrepository.subunit.gz"
state: file
register: local_subunit_file
- name: Check that HTML test result file has been pulled
delegate_to: localhost
file:
path: "{{ zuul.executor.log_root }}/testr_results.html.gz"
state: file
register: local_html_test_results
- name: Validate that files were pulled correctly
assert:
that:
- local_subunit_file is not changed
- local_subunit_file is succeeded
- local_html_test_results is not changed
- local_html_test_results is succeeded
# only one subunit file; the failed result should be hidden
- name: Check the content of the HTML file
delegate_to: localhost
shell: |
GLOBAL_RESULT=1
zgrep -q -E 'subunit_tests.test_working.WorkingTest.test_success$' \
{{ zuul.executor.log_root }}/testr_results.html.gz
T1=$?
zgrep -q -E 'subunit_tests.test_failing.FailingTest.test_failure.*_StringException:' \
{{ zuul.executor.log_root }}/testr_results.html.gz
T2=$?
if [ ${T1} -eq 0 ] && [ ${T2} -ne 0 ]; then
GLOBAL_RESULT=0
fi
exit $GLOBAL_RESULT
# The following test(s) require(s) the previous playbook
- name: Run the fetch-subunit-output role with multiple subunits
hosts: all
roles:
- role: fetch-subunit-output
zuul_additional_subunit_dirs:
- "/var/tmp/extratests"
post_tasks:
- name: Check that the testrepository file has been pulled
delegate_to: localhost
file:
path: "{{ zuul.executor.log_root }}/testrepository.subunit.gz"
state: file
register: local_subunit_file
- name: Check that HTML test result file has been pulled
delegate_to: localhost
file:
path: "{{ zuul.executor.log_root }}/testr_results.html.gz"
state: file
register: local_html_test_results
- name: Validate that files were pulled correctly
assert:
that:
- local_subunit_file is not changed
- local_subunit_file is succeeded
- local_html_test_results is not changed
- local_html_test_results is succeeded
- name: Check the content of the HTML file
delegate_to: localhost
shell: |
GLOBAL_RESULT=1
zgrep -q -E 'subunit_tests.test_working.WorkingTest.test_success$' \
{{ zuul.executor.log_root }}/testr_results.html.gz
T1=$?
zgrep -q -E 'subunit_tests.test_failing.FailingTest.test_failure.*_StringException:' \
{{ zuul.executor.log_root }}/testr_results.html.gz
T2=$?
if [ ${T1} -eq 0 ] && [ ${T2} -eq 0 ]; then
GLOBAL_RESULT=0
fi
exit $GLOBAL_RESULT

View File

@ -0,0 +1,21 @@
# Copyright 2019 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
class FailingTest(unittest.TestCase):
def test_failure(self):
self.assertTrue(False)

View File

@ -0,0 +1,21 @@
# Copyright 2019 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
class WorkingTest(unittest.TestCase):
def test_success(self):
self.assertTrue(True)

View File

@ -117,6 +117,7 @@
- ^roles/emit-job-header/.* - ^roles/emit-job-header/.*
- ^roles/ensure-output-dirs/.* - ^roles/ensure-output-dirs/.*
- ^roles/fetch-output/.* - ^roles/fetch-output/.*
- ^roles/fetch-subunit-output/.*
- ^roles/fetch-zuul-cloner/.* - ^roles/fetch-zuul-cloner/.*
- ^roles/set-zuul-log-path-fact/.* - ^roles/set-zuul-log-path-fact/.*
- ^roles/prepare-workspace-git/.* - ^roles/prepare-workspace-git/.*