Switch to pytest test runner

Change-Id: I8132e07ad624fe9faa01265d700b5eb37b94815f

roles/tobiko-check-collected-files/tasks/main.yaml

@@ -8,7 +8,6 @@
       - "tobiko.conf"
       - "tobiko.log"
       - "{{ test_report_name }}*.log"
-      - "{{ test_report_name }}*.subunit"
       - "{{ test_report_name }}*.html"
       - "{{ test_report_name }}*.xml"
   register: find_collected_files
@@ -33,7 +32,6 @@
         - tobiko.conf
         - '{{ test_report_name }}.log'
         - '{{ test_report_name }}.html'
-        - '{{ test_report_name }}.subunit'
         - '{{ test_report_name }}.xml'
 
   rescue:

roles/tobiko-common/defaults/main.yaml

@@ -46,7 +46,7 @@ test_report_files:
   - '{{ test_log_file | realpath }}'
 
 # Test report files prefix
-test_report_name: test_results
+test_report_name: tobiko_results
 
 # Log file where test cases should write logging messages to
 test_log_file: '{{ test_report_dir | realpath }}/tobiko.log'

setup.cfg

@@ -49,3 +49,6 @@ tag_svn_revision = 0
 
 [wheel]
 universal = 1
+
+[tool:pytest]
+render_collapsed = True

test-requirements.txt

@@ -1,9 +1,9 @@
 # Unit tests requirements
 
-coverage>=4.5.0        # Apache-2.0
-junitxml>=0.7.0        # LGPL-3
-mock>=2.0.0            # BSD
-os-testr>=1.0.0        # Apache-2.0
-psutil>=5.7.2          # BSD
-python-subunit>=1.4.0  # Apache-2.0
-stestr>=3.0.0          # Apache-2.0
+ansi2html>=1.6.0               # LGPLv3+
+coverage>=4.5.0                # Apache-2.0
+mock>=2.0.0                    # BSD
+psutil>=5.7.2                  # BSD
+pytest>=6.0.0                  # MIT
+pytest-html>=3.0.0             # MPL-2.0
+pytest-xdist[psutil]>=2.0.0    # MIT

tobiko/tests/conftest.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2020 Red Hat
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+from datetime import datetime
+
+from py.xml import html  # pylint: disable=no-name-in-module,import-error
+import pytest
+
+
+def pytest_html_results_table_header(cells):
+    cells.insert(2, html.th("Description"))
+    cells.insert(1, html.th("Time", class_="sortable time", col="time"))
+    cells.pop()
+
+
+def pytest_html_results_table_row(report, cells):
+    cells.insert(2, html.td(report.description))
+    cells.insert(1, html.td(datetime.utcnow(), class_="col-time"))
+    cells.pop()
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item, call):  # pylint: disable=unused-argument
+    outcome = yield
+    report = outcome.get_result()
+    report.description = str(item.function.__doc__)

tools/run_tests.py

@@ -29,26 +29,26 @@ from tools import common  # noqa
 
 LOG = common.get_logger(__name__)
 
+OS_TEST_PATH = common.normalize_path(os.environ['OS_TEST_PATH'])
 
 # Output dirs
 TOX_REPORT_DIR = common.normalize_path(
     os.environ.get('TOX_REPORT_DIR', os.getcwd()))
 
-TOX_REPORT_NAME = os.environ.get('TOX_REPORT_NAME', 'test_results')
+TOX_REPORT_NAME = os.environ.get('TOX_REPORT_NAME', 'tobiko_results')
 TOX_REPORT_PREFIX = os.path.join(TOX_REPORT_DIR, TOX_REPORT_NAME)
 
 TOX_REPORT_LOG = os.environ.get(
     'TOX_REPORT_LOG', TOX_REPORT_PREFIX + '.log')
 
-TOX_REPORT_SUBUNIT = os.environ.get(
-    'TOX_REPORT_SUBUNIT', TOX_REPORT_PREFIX + '.subunit')
-
 TOX_REPORT_HTML = os.environ.get(
     'TOX_REPORT_HTML', TOX_REPORT_PREFIX + '.html')
 
 TOX_REPORT_XML = os.environ.get(
     'TOX_REPORT_XML', TOX_REPORT_PREFIX + '.xml')
 
+TOX_NUM_PROCESSES = os.environ.get('TOX_NUM_PROCESSES') or 'auto'
+
 TOX_RUN_TESTS_TIMEOUT = float(os.environ.get('TOX_RUN_TESTS_TIMEOUT') or 0.)
 
 TOX_PYDEV_DEBUG = bool(
@@ -90,20 +90,6 @@ def run_tests():
         LOG.error(f"Error while running test cases.\n{ex}")
         succeeded = False
 
-    try:
-        log_tests_results()
-    except subprocess.CalledProcessError:
-        if succeeded:
-            raise
-
-    make_subunit_file()
-    make_html_file()
-    try:
-        make_xml_file()
-    except subprocess.CalledProcessError:
-        if succeeded:
-            raise
-
     return succeeded
 
 
@@ -137,8 +123,7 @@ def terminate_childs():
 
 
 def cleanup_report_dir():
-    for report_file in [TOX_REPORT_LOG, TOX_REPORT_SUBUNIT, TOX_REPORT_HTML,
-                        TOX_REPORT_XML]:
+    for report_file in [TOX_REPORT_LOG, TOX_REPORT_HTML, TOX_REPORT_XML]:
         if not common.make_dir(os.path.dirname(report_file)):
             common.remove_file(report_file)
 
@@ -148,13 +133,6 @@ def log_environ():
                    capture_stdout=False)
 
 
-def log_tests_results():
-    common.execute('stestr last --all-attachments >> "{log_file}"',
-                   log_file=TOX_REPORT_LOG,
-                   capture_stdout=False,
-                   check=False)
-
-
 def debug_test_cases():
     common.execute_python('-m testtools.run {posargs}',
                           posargs=common.get_posargs(),
@@ -162,28 +140,14 @@ def debug_test_cases():
 
 
 def run_test_cases():
-    common.execute('stestr run {posargs}',
-                   posargs=common.get_posargs(),
-                   capture_stdout=False)
-
-
-def make_subunit_file():
-    common.execute('stestr last --subunit > "{subunit_file}"',
-                   subunit_file=TOX_REPORT_SUBUNIT,
-                   capture_stdout=False)
-
-
-def make_html_file():
-    common.execute('subunit2html "{subunit_file}" "{html_file}"',
-                   subunit_file=TOX_REPORT_SUBUNIT,
-                   html_file=TOX_REPORT_HTML,
-                   capture_stdout=False)
-
-
-def make_xml_file():
-    common.execute('subunit2junitxml "{subunit_file}" -o "{xml_file}"',
-                   subunit_file=TOX_REPORT_SUBUNIT,
-                   xml_file=TOX_REPORT_XML,
+    xdist_options = ''
+    if TOX_NUM_PROCESSES != '1':
+        xdist_options = f"--numprocesses {TOX_NUM_PROCESSES} --dist loadscope"
+    common.execute(f"pytest "
+                   f"{xdist_options} "
+                   f"--junitxml={TOX_REPORT_XML} "
+                   f"--html={TOX_REPORT_HTML} --self-contained-html "
+                   f"{common.get_posargs()}",
                    capture_stdout=False)
 
 

tox.ini

@@ -27,10 +27,11 @@ setenv =
     OS_TEST_PATH = {toxinidir}/tobiko/tests/unit
     PS1 = [tobiko@{envname}] {env:PS1:}
     PYTHONWARNINGS = ignore::Warning,{env:PYTHONWARNINGS:}
-    RUN_TESTS_EXTRA_ARGS = --slowest
+    RUN_TESTS_EXTRA_ARGS = {env:OS_TEST_PATH}
     TOBIKO_PREVENT_CREATE = {env:TOBIKO_PREVENT_CREATE:false}
-    TOX_REPORT_NAME = {env:TOX_REPORT_NAME:test_results}
-    TOX_REPORT_DIR = {env:TOX_REPORT_DIR:{toxinidir}/report/{envname}}
+    TOX_NUM_PROCESSES = {env:TOX_NUM_PROCESSES:auto}
+    TOX_REPORT_NAME = {env:TOX_REPORT_NAME:tobiko_results_{envname}}
+    TOX_REPORT_DIR = {env:TOX_REPORT_DIR:{envlogdir}}
     TOX_CONSTRAINTS = {env:TOX_CONSTRAINTS:-c{env:TOX_CONSTRAINTS_FILE}}
     TOX_CONSTRAINTS_FILE = {env:TOX_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt}
     TOX_EXTRA_REQUIREMENTS = {env:TOX_EXTRA_REQUIREMENTS:-r{toxinidir}/extra-requirements.txt}
@@ -52,7 +53,6 @@ envdir = {[testenv:py3]envdir}
 setenv =
     {[testenv]setenv}
     PYTHON = coverage run --parallel-mode
-    RUN_TESTS_EXTRA_ARGS = --no-subunit-trace
     TOX_COVER_DIR={env:TOX_COVER_DIR:{toxinidir}/cover}
 commands =
     find . -type f -name ".coverage.*" -delete
@@ -61,7 +61,7 @@ commands =
     coverage combine
     coverage html -d "{env:TOX_COVER_DIR}"
     coverage xml -o "{env:TOX_COVER_DIR}/cover/coverage.xml"
-    coverage report --fail-under=40 --skip-covered
+    coverage report --skip-covered
     find . -type f -name ".coverage.*" -delete
 whitelist_externals =
     find
@@ -209,7 +209,7 @@ passenv = {[integration]passenv}
 setenv =
     {[integration]setenv}
     OS_TEST_PATH = {toxinidir}/tobiko/tests/faults
-    RUN_TESTS_EXTRA_ARGS = --serial
+    TOX_NUM_PROCESSES = 1
 
 
 [testenv:instanceha]
@@ -220,7 +220,8 @@ passenv = {[integration]passenv}
 setenv =
     {[integration]setenv}
     OS_TEST_PATH = {toxinidir}/tobiko/tests/faults/iha
-    RUN_TESTS_EXTRA_ARGS = --serial
+    TOX_NUM_PROCESSES = 1
+
 
 # --- CI workflow test environments -------------------------------------------