From 3ce3c741e5d7bf1eb59d5784d36955e68f52e72d Mon Sep 17 00:00:00 2001 From: Federico Ressi Date: Thu, 3 Dec 2020 14:34:41 +0100 Subject: [PATCH] Switch to pytest test runner Change-Id: I8132e07ad624fe9faa01265d700b5eb37b94815f --- .../tasks/main.yaml | 2 - roles/tobiko-common/defaults/main.yaml | 2 +- setup.cfg | 3 + test-requirements.txt | 14 ++--- tobiko/tests/conftest.py | 39 ++++++++++++ tools/run_tests.py | 62 ++++--------------- tox.ini | 15 ++--- 7 files changed, 71 insertions(+), 66 deletions(-) create mode 100644 tobiko/tests/conftest.py diff --git a/roles/tobiko-check-collected-files/tasks/main.yaml b/roles/tobiko-check-collected-files/tasks/main.yaml index 2dda162dc..c3af73a98 100644 --- a/roles/tobiko-check-collected-files/tasks/main.yaml +++ b/roles/tobiko-check-collected-files/tasks/main.yaml @@ -8,7 +8,6 @@ - "tobiko.conf" - "tobiko.log" - "{{ test_report_name }}*.log" - - "{{ test_report_name }}*.subunit" - "{{ test_report_name }}*.html" - "{{ test_report_name }}*.xml" register: find_collected_files @@ -33,7 +32,6 @@ - tobiko.conf - '{{ test_report_name }}.log' - '{{ test_report_name }}.html' - - '{{ test_report_name }}.subunit' - '{{ test_report_name }}.xml' rescue: diff --git a/roles/tobiko-common/defaults/main.yaml b/roles/tobiko-common/defaults/main.yaml index fe99c0d43..9c0d0570c 100644 --- a/roles/tobiko-common/defaults/main.yaml +++ b/roles/tobiko-common/defaults/main.yaml @@ -46,7 +46,7 @@ test_report_files: - '{{ test_log_file | realpath }}' # Test report files prefix -test_report_name: test_results +test_report_name: tobiko_results # Log file where test cases should write logging messages to test_log_file: '{{ test_report_dir | realpath }}/tobiko.log' diff --git a/setup.cfg b/setup.cfg index ca52986e9..53d1e34cf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,3 +49,6 @@ tag_svn_revision = 0 [wheel] universal = 1 + +[tool:pytest] +render_collapsed = True diff --git a/test-requirements.txt b/test-requirements.txt index 7c5f6fd9e..a46e3eebf 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,9 +1,9 @@ # Unit tests requirements -coverage>=4.5.0 # Apache-2.0 -junitxml>=0.7.0 # LGPL-3 -mock>=2.0.0 # BSD -os-testr>=1.0.0 # Apache-2.0 -psutil>=5.7.2 # BSD -python-subunit>=1.4.0 # Apache-2.0 -stestr>=3.0.0 # Apache-2.0 +ansi2html>=1.6.0 # LGPLv3+ +coverage>=4.5.0 # Apache-2.0 +mock>=2.0.0 # BSD +psutil>=5.7.2 # BSD +pytest>=6.0.0 # MIT +pytest-html>=3.0.0 # MPL-2.0 +pytest-xdist[psutil]>=2.0.0 # MIT diff --git a/tobiko/tests/conftest.py b/tobiko/tests/conftest.py new file mode 100644 index 000000000..15df3e00a --- /dev/null +++ b/tobiko/tests/conftest.py @@ -0,0 +1,39 @@ +# Copyright (c) 2020 Red Hat +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +from datetime import datetime + +from py.xml import html # pylint: disable=no-name-in-module,import-error +import pytest + + +def pytest_html_results_table_header(cells): + cells.insert(2, html.th("Description")) + cells.insert(1, html.th("Time", class_="sortable time", col="time")) + cells.pop() + + +def pytest_html_results_table_row(report, cells): + cells.insert(2, html.td(report.description)) + cells.insert(1, html.td(datetime.utcnow(), class_="col-time")) + cells.pop() + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): # pylint: disable=unused-argument + outcome = yield + report = outcome.get_result() + report.description = str(item.function.__doc__) diff --git a/tools/run_tests.py b/tools/run_tests.py index 444a2bec8..e9f69c54a 100755 --- a/tools/run_tests.py +++ b/tools/run_tests.py @@ -29,26 +29,26 @@ from tools import common # noqa LOG = common.get_logger(__name__) +OS_TEST_PATH = common.normalize_path(os.environ['OS_TEST_PATH']) # Output dirs TOX_REPORT_DIR = common.normalize_path( os.environ.get('TOX_REPORT_DIR', os.getcwd())) -TOX_REPORT_NAME = os.environ.get('TOX_REPORT_NAME', 'test_results') +TOX_REPORT_NAME = os.environ.get('TOX_REPORT_NAME', 'tobiko_results') TOX_REPORT_PREFIX = os.path.join(TOX_REPORT_DIR, TOX_REPORT_NAME) TOX_REPORT_LOG = os.environ.get( 'TOX_REPORT_LOG', TOX_REPORT_PREFIX + '.log') -TOX_REPORT_SUBUNIT = os.environ.get( - 'TOX_REPORT_SUBUNIT', TOX_REPORT_PREFIX + '.subunit') - TOX_REPORT_HTML = os.environ.get( 'TOX_REPORT_HTML', TOX_REPORT_PREFIX + '.html') TOX_REPORT_XML = os.environ.get( 'TOX_REPORT_XML', TOX_REPORT_PREFIX + '.xml') +TOX_NUM_PROCESSES = os.environ.get('TOX_NUM_PROCESSES') or 'auto' + TOX_RUN_TESTS_TIMEOUT = float(os.environ.get('TOX_RUN_TESTS_TIMEOUT') or 0.) TOX_PYDEV_DEBUG = bool( @@ -90,20 +90,6 @@ def run_tests(): LOG.error(f"Error while running test cases.\n{ex}") succeeded = False - try: - log_tests_results() - except subprocess.CalledProcessError: - if succeeded: - raise - - make_subunit_file() - make_html_file() - try: - make_xml_file() - except subprocess.CalledProcessError: - if succeeded: - raise - return succeeded @@ -137,8 +123,7 @@ def terminate_childs(): def cleanup_report_dir(): - for report_file in [TOX_REPORT_LOG, TOX_REPORT_SUBUNIT, TOX_REPORT_HTML, - TOX_REPORT_XML]: + for report_file in [TOX_REPORT_LOG, TOX_REPORT_HTML, TOX_REPORT_XML]: if not common.make_dir(os.path.dirname(report_file)): common.remove_file(report_file) @@ -148,13 +133,6 @@ def log_environ(): capture_stdout=False) -def log_tests_results(): - common.execute('stestr last --all-attachments >> "{log_file}"', - log_file=TOX_REPORT_LOG, - capture_stdout=False, - check=False) - - def debug_test_cases(): common.execute_python('-m testtools.run {posargs}', posargs=common.get_posargs(), @@ -162,28 +140,14 @@ def debug_test_cases(): def run_test_cases(): - common.execute('stestr run {posargs}', - posargs=common.get_posargs(), - capture_stdout=False) - - -def make_subunit_file(): - common.execute('stestr last --subunit > "{subunit_file}"', - subunit_file=TOX_REPORT_SUBUNIT, - capture_stdout=False) - - -def make_html_file(): - common.execute('subunit2html "{subunit_file}" "{html_file}"', - subunit_file=TOX_REPORT_SUBUNIT, - html_file=TOX_REPORT_HTML, - capture_stdout=False) - - -def make_xml_file(): - common.execute('subunit2junitxml "{subunit_file}" -o "{xml_file}"', - subunit_file=TOX_REPORT_SUBUNIT, - xml_file=TOX_REPORT_XML, + xdist_options = '' + if TOX_NUM_PROCESSES != '1': + xdist_options = f"--numprocesses {TOX_NUM_PROCESSES} --dist loadscope" + common.execute(f"pytest " + f"{xdist_options} " + f"--junitxml={TOX_REPORT_XML} " + f"--html={TOX_REPORT_HTML} --self-contained-html " + f"{common.get_posargs()}", capture_stdout=False) diff --git a/tox.ini b/tox.ini index c83de5d6d..a6a2a1e12 100644 --- a/tox.ini +++ b/tox.ini @@ -27,10 +27,11 @@ setenv = OS_TEST_PATH = {toxinidir}/tobiko/tests/unit PS1 = [tobiko@{envname}] {env:PS1:} PYTHONWARNINGS = ignore::Warning,{env:PYTHONWARNINGS:} - RUN_TESTS_EXTRA_ARGS = --slowest + RUN_TESTS_EXTRA_ARGS = {env:OS_TEST_PATH} TOBIKO_PREVENT_CREATE = {env:TOBIKO_PREVENT_CREATE:false} - TOX_REPORT_NAME = {env:TOX_REPORT_NAME:test_results} - TOX_REPORT_DIR = {env:TOX_REPORT_DIR:{toxinidir}/report/{envname}} + TOX_NUM_PROCESSES = {env:TOX_NUM_PROCESSES:auto} + TOX_REPORT_NAME = {env:TOX_REPORT_NAME:tobiko_results_{envname}} + TOX_REPORT_DIR = {env:TOX_REPORT_DIR:{envlogdir}} TOX_CONSTRAINTS = {env:TOX_CONSTRAINTS:-c{env:TOX_CONSTRAINTS_FILE}} TOX_CONSTRAINTS_FILE = {env:TOX_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt} TOX_EXTRA_REQUIREMENTS = {env:TOX_EXTRA_REQUIREMENTS:-r{toxinidir}/extra-requirements.txt} @@ -52,7 +53,6 @@ envdir = {[testenv:py3]envdir} setenv = {[testenv]setenv} PYTHON = coverage run --parallel-mode - RUN_TESTS_EXTRA_ARGS = --no-subunit-trace TOX_COVER_DIR={env:TOX_COVER_DIR:{toxinidir}/cover} commands = find . -type f -name ".coverage.*" -delete @@ -61,7 +61,7 @@ commands = coverage combine coverage html -d "{env:TOX_COVER_DIR}" coverage xml -o "{env:TOX_COVER_DIR}/cover/coverage.xml" - coverage report --fail-under=40 --skip-covered + coverage report --skip-covered find . -type f -name ".coverage.*" -delete whitelist_externals = find @@ -209,7 +209,7 @@ passenv = {[integration]passenv} setenv = {[integration]setenv} OS_TEST_PATH = {toxinidir}/tobiko/tests/faults - RUN_TESTS_EXTRA_ARGS = --serial + TOX_NUM_PROCESSES = 1 [testenv:instanceha] @@ -220,7 +220,8 @@ passenv = {[integration]passenv} setenv = {[integration]setenv} OS_TEST_PATH = {toxinidir}/tobiko/tests/faults/iha - RUN_TESTS_EXTRA_ARGS = --serial + TOX_NUM_PROCESSES = 1 + # --- CI workflow test environments -------------------------------------------