Refactor test execution scripts

- Acreate a new tools/run_tests.py script to
  execute test cases
- Move report generation to such script and
  remove 'report' tox environment
- Running test cases generates report files
  (.log, .html, .xml) into 'report/{envname}'
  directory
- Test cases failures will returns exit code 1
  Other script test runner failures returns
  exit code 2

Big refactory to tox.ini file to try semplifying it:

- create the new 'py3' tox environment for running unit
  tests as default platform python 3 interpreter
- use same as py3 '{envdir}' for scenario, functional,
  neutron and faults tox environments

Change-Id: Id09425245cc86b84b41e6b3b1c1db759cc686f3a

.gitignore

@@ -21,11 +21,11 @@ __pycache__
 AUTHORS
 build/*
 ChangeLog
-cover/
 doc/build/*
+cover/
 dist/
 etc/*.sample
-tobiko_results*
+report/
 test_results*
 zuul/versioninfo
 

report-requirements.txt

@@ -1,7 +0,0 @@
-# Requirements file for 'report' Tox environment
-
-junitxml>=0.7                                  # LGPL-3
-os-testr>=1.0                                  # Apache-2.0
-python-subunit>=1.4 ; python_version >= '3.0'  # Apache-2.0
-python-subunit<1.4 ; python_version < '3.0'    # Apache-2.0
-stestr>=2.0                                    # Apache-2.0

roles/tobiko-check-collected-files/tasks/main.yaml

@@ -31,6 +31,7 @@
       loop:
         - tobiko.log
         - tobiko.conf
+        - '{{ test_report_name }}.log'
         - '{{ test_report_name }}.html'
         - '{{ test_report_name }}.subunit'
         - '{{ test_report_name }}.xml'

roles/tobiko-common/defaults/main.yaml

@@ -52,7 +52,7 @@ test_conf: {}
 # --- Test report options -----------------------------------------------------
 
 # Remote directory where test cases shoulw write report files to
-test_report_dir: "{{ test_dir | realpath }}"
+test_report_dir: "{{ test_dir | realpath }}/report"
 test_report_files:
   - '{{ test_conf_file | realpath }}'
   - '{{ test_log_file | realpath }}'

roles/tobiko-ensure-tox/defaults/main.yaml

@@ -1,4 +1,4 @@
 ---
 
 tox_command: tox
-tox_min_version: 3.4
+tox_min_version: 3.8

roles/tobiko-tox/defaults/main.yaml

@@ -14,11 +14,10 @@ tox_step_name:
 tox_step_index: 0
 tox_report_name:
   "{{ test_report_name }}{% if tox_step_index %}_{{ '{:02d}'.format(tox_step_index | int) }}{% endif %}{% if tox_step_name %}_{{ tox_step_name }}{% endif %}{% if tox_envlist %}_{{ tox_envlist }}{% endif %}"
-tox_report_env:
-  TOBIKO_TEST_REPORT_DIR: '{{ tox_report_dir }}'
-  TOBIKO_TEST_REPORT_NAME: '{{ tox_report_name }}'
 
 tox_constraints_file: '{{ remote_constraints_file }}'
 
 tox_constrain_env:
-  UPPER_CONSTRAINTS_FILE: '{{ tox_constraints_file }}'
+  TOX_REPORT_DIR: '{{ tox_report_dir }}'
+  TOX_REPORT_NAME: '{{ tox_report_name }}'
+  TOX_CONSTRAINTS_FILE: '{{ tox_constraints_file }}'

roles/tobiko-tox/tasks/tox.yaml

@@ -20,7 +20,6 @@
       tox_description: '{{ tox_description }}'
       tox_dir: '{{ tox_dir }}'
       tox_environment: '{{ tox_environment | combine(tox_constrain_env) }}'
-      tox_report_env: '{{ tox_report_env | combine(tox_constrain_env) }}'
 
 
 - name: "{{ tox_description }}"
@@ -39,17 +38,6 @@
     - (run_tox.stdout_lines | length) > 0
 
 
-- name: "generate test case report files"
-  shell:
-    chdir: "{{ tobiko_dir }}"
-    cmd: |
-        {{ tox_command }} -e report
-  register:
-    make_report
-  environment: '{{ tox_report_env | combine(tox_constrain_env) }}'
-  ignore_errors: yes
-
-
 - name:
   block:
     - name: "show test cases errors"

test-requirements.txt

@@ -1,5 +1,9 @@
 # Unit tests requirements
 
-coverage!=4.4,>=4.0     # Apache-2.0
-mock>=2.0               # BSD
-testscenarios>=0.4      # Apache-2.0/BSD
+coverage!=4.4,>=4.0                            # Apache-2.0
+junitxml>=0.7                                  # LGPL-3
+mock>=2.0                                      # BSD
+os-testr>=1.0                                  # Apache-2.0
+python-subunit>=1.4 ; python_version >= '3.0'  # Apache-2.0
+python-subunit<1.4 ; python_version < '3.0'    # Apache-2.0
+stestr>=2.0                                    # Apache-2.0

tools/common.py

@@ -0,0 +1,95 @@
+# Copyright 2018 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+import logging
+import os
+import shlex
+import subprocess
+import sys
+import tempfile
+
+
+LOG = logging.getLogger(__name__)
+
+
+def get_logger(name):
+    module = sys.modules.get(name)
+    if module:
+        name = name_from_path(module.__file__)
+    return logging.getLogger(name)
+
+
+def setup_logging(main_script=None, level=logging.DEBUG):
+    main_script = main_script or sys.modules['__main__'].__file__
+    logging.basicConfig(
+        level=level,
+        stream=sys.stderr,
+        format='%(name)-s: %(levelname)-7s %(asctime)-15s | %(message)s')
+    return logging.getLogger(name=name_from_path(main_script))
+
+
+def name_from_path(path):
+    return os.path.splitext(os.path.basename(path))[0]
+
+
+def normalize_path(path):
+    return os.path.realpath(os.path.expanduser(path))
+
+
+def execute(command, *args, **kwargs):
+    capture_stdout = kwargs.pop('capture_stdout', True)
+    universal_newlines = kwargs.pop('universal_newlines', True)
+
+    if args or kwargs:
+        command = command.format(*args, **kwargs)
+    command = command.strip()
+
+    if capture_stdout:
+        execute_func = subprocess.check_output
+    else:
+        execute_func = subprocess.check_call
+
+    return execute_func(['/bin/bash', '-x', '-c', command],
+                        shell=False, universal_newlines=universal_newlines)
+
+
+def get_posargs(args=None):
+    if args is None:
+        args = sys.argv[1:]
+    return ' '.join(shlex.quote(s) for s in args)
+
+
+def make_temp(*args, **kwargs):
+    fd, filename = tempfile.mkstemp(*args, **kwargs)
+    os.close(fd)
+    return filename
+
+
+def make_dir(dirname):
+    if os.path.isdir(dirname):
+        return False
+    else:
+        LOG.debug("Create directory: '%s'", dirname)
+        os.makedirs(dirname)
+        return True
+
+
+def remove_file(filename):
+    if os.path.isfile(filename):
+        LOG.debug("Remove file: '%s'", filename)
+        os.unlink(filename)
+        return True
+    else:
+        return False

tools/run_tests.py

@@ -0,0 +1,137 @@
+# Copyright 2018 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+import os
+import sys
+import subprocess
+
+
+TOP_DIR = os.path.dirname(os.path.dirname(__file__))
+if TOP_DIR not in sys.path:
+    sys.path.insert(0, TOP_DIR)
+
+from tools import common  # noqa
+
+LOG = common.get_logger(__name__)
+
+
+# Output dirs
+TOX_REPORT_DIR = common.normalize_path(
+    os.environ.get('TOX_REPORT_DIR', os.getcwd()))
+
+TOX_REPORT_NAME = os.environ.get('TOX_REPORT_NAME', 'test_results')
+TOX_REPORT_PREFIX = os.path.join(TOX_REPORT_DIR, TOX_REPORT_NAME)
+
+TOX_REPORT_LOG = os.environ.get(
+    'TOX_REPORT_LOG', TOX_REPORT_PREFIX + '.log')
+
+TOX_REPORT_SUBUNIT = os.environ.get(
+    'TOX_REPORT_SUBUNIT', TOX_REPORT_PREFIX + '.subunit')
+
+TOX_REPORT_HTML = os.environ.get(
+    'TOX_REPORT_HTML', TOX_REPORT_PREFIX + '.html')
+
+TOX_REPORT_XML = os.environ.get(
+    'TOX_REPORT_XML', TOX_REPORT_PREFIX + '.xml')
+
+
+def main():
+    common.setup_logging()
+    try:
+        succeeded = run_tests()
+        if succeeded:
+            LOG.info('SUCCEEDED')
+        else:
+            LOG.info('FAILED')
+            sys.exit(1)
+
+    except Exception:
+        LOG.exception('ERROR')
+        sys.exit(2)
+
+
+def run_tests():
+    cleanup_report_dir()
+    log_environ()
+
+    succeeded = True
+    try:
+        run_test_cases()
+    except subprocess.CalledProcessError:
+        succeeded = False
+
+    try:
+        log_tests_results()
+    except subprocess.CalledProcessError:
+        if succeeded:
+            raise
+
+    make_subunit_file()
+    make_html_file()
+    try:
+        make_xml_file()
+    except subprocess.CalledProcessError:
+        if succeeded:
+            raise
+
+    return succeeded
+
+
+def cleanup_report_dir():
+    for report_file in [TOX_REPORT_LOG, TOX_REPORT_SUBUNIT, TOX_REPORT_HTML,
+                        TOX_REPORT_XML]:
+        if not common.make_dir(os.path.dirname(report_file)):
+            common.remove_file(report_file)
+
+
+def log_environ():
+    common.execute('env | sort >> "{log_file}"', log_file=TOX_REPORT_LOG,
+                   capture_stdout=False)
+
+
+def log_tests_results():
+    common.execute('stestr last --all-attachments >> "{log_file}"',
+                   log_file=TOX_REPORT_LOG,
+                   capture_stdout=False)
+
+
+def run_test_cases():
+    common.execute('stestr run --slowest {posargs}',
+                   posargs=common.get_posargs(),
+                   capture_stdout=False)
+
+
+def make_subunit_file():
+    common.execute('stestr last --subunit > "{subunit_file}"',
+                   subunit_file=TOX_REPORT_SUBUNIT,
+                   capture_stdout=False)
+
+
+def make_html_file():
+    common.execute('subunit2html "{subunit_file}" "{html_file}"',
+                   subunit_file=TOX_REPORT_SUBUNIT,
+                   html_file=TOX_REPORT_HTML,
+                   capture_stdout=False)
+
+
+def make_xml_file():
+    common.execute('subunit2junitxml "{subunit_file}" -o "{xml_file}"',
+                   subunit_file=TOX_REPORT_SUBUNIT,
+                   xml_file=TOX_REPORT_XML,
+                   capture_stdout=False)
+
+
+if __name__ == '__main__':
+    main()

tools/setup_infrared.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Red Hat
+# Copyright 2020 Red Hat
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -13,29 +13,26 @@
 #    under the License.
 from __future__ import absolute_import
 
-import logging
 import os
 import subprocess
 import sys
 
+TOP_DIR = os.path.dirname(os.path.dirname(__file__))
+if TOP_DIR not in sys.path:
+    sys.path.insert(0, TOP_DIR)
 
-LOG = logging.getLogger(__name__)
+from tools import common  # noqa
+
+LOG = common.get_logger(__name__)
 
 
 def main():
-    setup_logging()
+    common.setup_logging()
     add_tobiko_plugin()
     ensure_workspace()
     copy_inventory()
 
 
-def setup_logging(level=logging.DEBUG):
-    logging.basicConfig(
-        level=level,
-        stream=sys.stderr,
-        format='%(name)-s: %(levelname)-7s %(asctime)-15s | %(message)s')
-
-
 def add_tobiko_plugin(path=None):
     path = path or os.environ.get('IR_TOBIKO_PLUGIN')
     if path:
@@ -43,19 +40,19 @@ def add_tobiko_plugin(path=None):
 
 
 def add_plugin(name, path):
-    path = normalize_path(path)
+    path = common.normalize_path(path)
     if not os.path.isdir(path):
         message = ("invalid plug-in '{}' directory: '{}'").format(name, path)
         raise RuntimeError(message)
 
     remove_plugin(name)
-    execute('ir plugin add "{}"', path)
+    common.execute('ir plugin add "{}"', path)
     LOG.info("plug-in '%s' added from path '%s'", name, path)
 
 
 def remove_plugin(name):
     try:
-        execute('ir plugin remove "{}"', name)
+        common.execute('ir plugin remove "{}"', name)
     except subprocess.CalledProcessError as ex:
         LOG.debug("plug-in '%s' not removed: %s", name, ex)
         return False
@@ -68,11 +65,11 @@ def ensure_workspace(filename=None):
     filename = (filename or
                 os.environ.get('IR_WORKSPACE_FILE') or
                 'workspace.tgz')
-    filename = normalize_path(filename)
-    workspace = name_from_path(filename)
+    filename = common.normalize_path(filename)
+    workspace = common.name_from_path(filename)
     if os.path.isfile(filename):
         try:
-            execute('ir workspace import "{}"', filename)
+            common.execute('ir workspace import "{}"', filename)
         except subprocess.CalledProcessError as ex:
             LOG.debug("workspace file '%s' not imported: %s", filename, ex)
         else:
@@ -82,14 +79,14 @@ def ensure_workspace(filename=None):
         LOG.debug("workspace file not found: '%s'", filename)
 
     try:
-        execute('ir workspace checkout "{}"', workspace)
+        common.execute('ir workspace checkout "{}"', workspace)
     except subprocess.CalledProcessError as ex:
         LOG.debug("workspace '%s' not checked out: %s", workspace, ex)
     else:
         LOG.info("workspace '%s' checked out", workspace)
         return
 
-    execute('infrared workspace checkout --create "{}"', workspace)
+    common.execute('infrared workspace checkout --create "{}"', workspace)
     LOG.info("workspace '%s' created", workspace)
 
 
@@ -101,7 +98,7 @@ def copy_inventory(filename=None):
         LOG.debug('inventary file not found: %r', filename)
         return False
 
-    dest_file = execute('ir workspace inventory')
+    dest_file = common.execute('ir workspace inventory')
     LOG.debug("got workspace inventory file: '%s'", dest_file)
 
     dest_dir = os.path.basename(dest_file)
@@ -109,27 +106,10 @@ def copy_inventory(filename=None):
         os.makedirs(dest_dir)
         LOG.info("directory created: '%s'", dest_dir)
 
-    execute('cp {} {}', filename, dest_file)
+    common.execute('cp {} {}', filename, dest_file)
     LOG.info("inventary file '%s' copied to '%s'", filename, dest_file)
     return True
 
 
-def normalize_path(path):
-    return os.path.realpath(os.path.expanduser(path))
-
-
-def execute(command, *args, **kwargs):
-    if args or kwargs:
-        command = command.format(*args, **kwargs)
-    LOG.debug("execute command: '%s'", command)
-    return subprocess.check_output(command, shell=True,
-                                   universal_newlines=True)
-
-
-def name_from_path(path):
-    return os.path.splitext(os.path.basename(path))[0]
-
-
 if __name__ == '__main__':
-    LOG = logging.getLogger(name_from_path(__file__))
     main()

tox.ini

@@ -1,59 +1,63 @@
 [tox]
 
-envlist = bindep,linters,cover
-minversion = 3.4.0
+envlist = bindep,linters,py3
+minversion = 3.8.0
 
 
 # --- unit test environments -------------------------------------------------
 
-[base]
-
-deps =
-    -c{env:UPPER_CONSTRAINTS_FILE}
-    -r{toxinidir}/requirements.txt
-passenv =
-    PS1
-    TOBIKO_*
-setenv =
-    OS_LOG_CAPTURE={env:OS_LOG_CAPTURE:true}
-    OS_STDOUT_CAPTURE={env:OS_STDOUT_CAPTURE:true}
-    OS_STDERR_CAPTURE={env:OS_STDERR_CAPTURE:true}
-    PS1=[tobiko@{envname}] {env:PS1:}
-    PYTHON={env:PYTHON:python3}
-    PYTHONWARNINGS=ignore::Warning,{env:PYTHONWARNINGS:}
-    TOBIKO_PREVENT_CREATE={env:TOBIKO_PREVENT_CREATE:false}
-    VIRTUAL_ENV={envdir}
-    UPPER_CONSTRAINTS_FILE={env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt}
-commands =
-    stestr run {posargs}
-
-
 [testenv]
 
 usedevelop = True
 deps =
-    {[base]deps}
+    -c{env:TOX_CONSTRAINTS_FILE}
+    -r{toxinidir}/requirements.txt
     -r{toxinidir}/test-requirements.txt
-passenv = {[base]passenv}
+passenv =
+    JENKINS_*
+    PS1
+    OS_*
+    TOBIKO_*
+    TOX_*
 setenv =
-    {[base]setenv}
-    OS_TEST_PATH={toxinidir}/tobiko/tests/unit
-commands = {[base]commands}
+    OS_LOG_CAPTURE = {env:OS_LOG_CAPTURE:true}
+    OS_STDOUT_CAPTURE = {env:OS_STDOUT_CAPTURE:true}
+    OS_STDERR_CAPTURE = {env:OS_STDERR_CAPTURE:true}
+    OS_TEST_PATH = {toxinidir}/tobiko/tests/unit
+    PS1 = [tobiko@{envname}] {env:PS1:}
+    PYTHONWARNINGS = ignore::Warning,{env:PYTHONWARNINGS:}
+    RUN_TESTS_EXTRA_ARGS =
+    TOBIKO_PREVENT_CREATE = {env:TOBIKO_PREVENT_CREATE:false}
+    TOX_REPORT_NAME = {env:TOX_REPORT_NAME:test_results}
+    TOX_REPORT_DIR = {env:TOX_REPORT_DIR:{toxinidir}/report/{envname}}
+    TOX_CONSTRAINTS_FILE = {env:TOX_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt}
+    VIRTUAL_ENV = {envdir}
+commands =
+    {envpython} {toxinidir}/tools/run_tests.py {env:RUN_TESTS_EXTRA_ARGS} {posargs}
+
+
+[testenv:py3]
+
+basepython = {env:TOX_PYTHON:python3}
+envdir = {toxworkdir}/py3
 
 
 [testenv:cover]
 
-basepython = python3
+basepython = {[testenv:py3]basepython}
+envdir = {[testenv:py3]envdir}
 setenv =
     {[testenv]setenv}
-    PYTHON=coverage run --parallel-mode
+    PYTHON = coverage run --parallel-mode
+    RUN_TESTS_EXTRA_ARGS = --no-subunit-trace
+    TOX_COVER_DIR={env:TOX_COVER_DIR:{toxinidir}/cover}
 commands =
     find . -type f -name ".coverage.*" -delete
     coverage erase
-    stestr run --no-subunit-trace {posargs}
+    {[testenv]commands}
     coverage combine
-    coverage html -d cover
-    coverage xml -o cover/coverage.xml
+    coverage html -d "{env:TOX_COVER_DIR}"
+    coverage xml -o "{env:TOX_COVER_DIR}/cover/coverage.xml"
     coverage report --fail-under=40 --skip-covered
     find . -type f -name ".coverage.*" -delete
 whitelist_externals =
@@ -62,10 +66,9 @@ whitelist_externals =
 
 # --- static analisys environments -------------------------------------------
 
-
 [testenv:pep8]
 
-basepython = {env:PYTHON:python3}
+basepython = python3
 deps =
     {[testenv]deps}
     -r{toxinidir}/linters-requirements.txt
@@ -91,6 +94,7 @@ commands =
     pylint -j0 --max-line-length=80 -E -e W,E \
         -d unused-import,broad-except,fixme tobiko
 
+
 [flake8]
 # H106: Don't put vim configuration in source files
 # H203: Use assertIs(Not)None to check for None
@@ -106,76 +110,68 @@ import-order-style = pep8
 
 # --- integration test environments ------------------------------------------
 
-[openstack]
+[integration]
 
-basepython = {env:PYTHON:python3}
-deps =
-    {[base]deps}
+basepython = {[testenv:py3]basepython}
+envdir = {[testenv:py3]envdir}
 passenv =
-    {[base]passenv}
+    {[testenv]passenv}
     *_proxy
-    OS_*
-setenv =
-    {[base]setenv}
-    ANSIBLE_CONFIG={env:ANSIBLE_CONFIG:{toxinidir}/ansible.cfg}
-commands = {[base]commands}
-
-
-[testenv:functional]
-
-basepython = {[openstack]basepython}
-deps = {[openstack]deps}
-passenv = {[openstack]passenv}
-setenv =
-    {[openstack]setenv}
-    OS_TEST_PATH={toxinidir}/tobiko/tests/functional
-
-
-[testenv:scenario]
-
-basepython = {[openstack]basepython}
-deps = {[openstack]deps}
-passenv = {[openstack]passenv}
-setenv =
-    {[openstack]setenv}
-    OS_TEST_PATH={toxinidir}/tobiko/tests/scenario
-
-
-[testenv:neutron]
-
-basepython = {[openstack]basepython}
-envdir = {toxworkdir}/scenario
-deps = {[testenv:scenario]deps}
-passenv = {[testenv:scenario]passenv}
-setenv =
-    {[testenv:scenario]setenv}
-    OS_TEST_PATH={toxinidir}/tobiko/tests/scenario/neutron
-
-
-[testenv:faults]
-
-basepython = {[openstack]basepython}
-envdir = {toxworkdir}/scenario
-deps = {[testenv:scenario]deps}
-passenv = {[testenv:scenario]passenv}
-setenv =
-    {[testenv:scenario]setenv}
-    OS_TEST_PATH={toxinidir}/tobiko/tests/faults
-commands =
-    stestr run --serial {posargs}
+setenv = {[testenv]setenv}
 
 
 [testenv:venv]
 
-basepython = {[openstack]basepython}
-envdir = {toxworkdir}/scenario
-deps = {[testenv:scenario]deps}
-passenv = {[testenv:scenario]passenv}
-setenv = {[testenv:scenario]setenv}
+basepython = {[integration]basepython}
+envdir = {[integration]envdir}
+passenv = {[integration]passenv}
 whitelist_externals = *
 commands = {posargs:bash}
 
 
+[testenv:functional]
+
+basepython = {[integration]basepython}
+envdir = {[integration]envdir}
+passenv = {[integration]passenv}
+setenv =
+    {[integration]setenv}
+    OS_TEST_PATH = {toxinidir}/tobiko/tests/functional
+
+
+[testenv:scenario]
+
+basepython = {[integration]basepython}
+envdir = {[integration]envdir}
+passenv = {[integration]passenv}
+setenv =
+    {[integration]setenv}
+    OS_TEST_PATH = {toxinidir}/tobiko/tests/scenario
+
+
+[testenv:neutron]
+
+basepython = {[integration]basepython}
+envdir = {[integration]envdir}
+passenv = {[integration]passenv}
+setenv =
+    {[integration]setenv}
+    OS_TEST_PATH = {toxinidir}/tobiko/tests/scenario/neutron
+
+
+[testenv:faults]
+
+basepython = {[integration]basepython}
+envdir = {[integration]envdir}
+passenv = {[integration]passenv}
+setenv =
+    {[integration]setenv}
+    OS_TEST_PATH = {toxinidir}/tobiko/tests/faults
+    RUN_TESTS_EXTRA_ARGS = --serial
+
+
+# --- CI workflow test environments -------------------------------------------
+
 [testenv:infrared]
 
 # On RedHat Linux must use the default unversioned python because of dependency on native SELinux
@@ -189,43 +185,40 @@ whitelist_externals =
     rm
 
 deps =
-    -c{env:UPPER_CONSTRAINTS_FILE}
+    -c{env:TOX_CONSTRAINTS_FILE}
     -rinfrared-requirements.txt
 passenv =
-    {[testenv:venv]passenv}
+    {[testenv]passenv}
     ANSIBLE_*
     IR_*
-    PYTHON
     TERM
 
 setenv =
-    {[testenv:venv]setenv}
-    ANSIBLE_CONFIG={env:ANSIBLE_CONFIG:{toxinidir}/ansible.cfg}
-    ANSIBLE_INVENTORY={env:ANSIBLE_INVENTORY:{toxinidir}/ansible_hosts}
-    IR_HOME={env:IR_HOME:{envdir}/home/infrared}
-    IR_TOBIKO_PLUGIN={env:IR_TOBIKO_PLUGIN:{toxinidir}/infrared_plugin}
-    IR_WORKSPACE_FILE={env:IR_WORKSPACE_FILE:{toxinidir}/workspace.tgz}
+    {[testenv]setenv}
+    ANSIBLE_CONFIG = {env:ANSIBLE_CONFIG:{toxinidir}/ansible.cfg}
+    ANSIBLE_INVENTORY = {env:ANSIBLE_INVENTORY:{toxinidir}/ansible_hosts}
+    IR_HOME = {env:IR_HOME:{envdir}/home/infrared}
+    IR_TOBIKO_PLUGIN = {env:IR_TOBIKO_PLUGIN:{toxinidir}/infrared_plugin}
+    IR_WORKSPACE_FILE = {env:IR_WORKSPACE_FILE:{toxinidir}/workspace.tgz}
 
 commands_pre =
     {envpython} {toxinidir}/tools/setup_infrared.py
 
 commands =
-    rm -fR '{toxinidir}/test_results'
-    ir tobiko \
-      --tobiko-src-dir {toxinidir} \
-      --collect-dir '{toxinidir}/test_results' \
-      --upper-constraints '{env:UPPER_CONSTRAINTS_FILE}' \
-      {posargs}
+    ir tobiko --tobiko-src-dir '{toxinidir}' \
+              --collect-dir '{env:TOX_REPORT_DIR}' \
+              --upper-constraints '{env:TOX_CONSTRAINTS_FILE}' \
+              {posargs}
 
 
 # --- documentation environments ----------------------------------------------
 
 [docs]
 
-basepython = {env:PYTHON:python3}
+basepython = {[testenv:py3]basepython}
 envdir = {toxworkdir}/docs
 deps =
-    {[base]deps}
+    {[testenv]deps}
     -r{toxinidir}/doc/requirements.txt
 
 
@@ -257,20 +250,9 @@ commands =
 
 # --- CI report environments --------------------------------------------------
 
-[testenv:report]
-
-basepython = {env:PYTHON:python3}
-usedevelop = false
-skipdist = true
-skip_install = true
-deps = -r {toxinidir}/report-requirements.txt
-passenv = {[base]passenv}
-setenv = {[base]setenv}
-commands = {toxinidir}/tools/ci/make_report
-
 
 [testenv:bindep]
-basepython = {env:PYTHON:python3}
+basepython = {[testenv:py3]basepython}
 # Do not install any requirements. We want this to be fast and work even if
 # system dependencies are missing, since it's used to tell you what system
 # dependencies are missing! This also means that bindep must be installed