x/tobiko
Code Issues Proposed changes

Switch to pytest test runner

Browse Source 
Change-Id: I8132e07ad624fe9faa01265d700b5eb37b94815f
This commit is contained in:
Federico Ressi 2020-12-03 14:34:41 +01:00
parent 27e0ce5ef3
commit 3ce3c741e5
7 changed files with 71 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
roles/tobiko-check-collected-files/tasks/main.yaml
View File

@ -8,7 +8,6 @@
- "tobiko.conf"
- "tobiko.log"
- "{{ test_report_name }}*.log"
- "{{ test_report_name }}*.subunit"
- "{{ test_report_name }}*.html"
- "{{ test_report_name }}*.xml"
register: find_collected_files
@ -33,7 +32,6 @@
- tobiko.conf
- '{{ test_report_name }}.log'
- '{{ test_report_name }}.html'
- '{{ test_report_name }}.subunit'
- '{{ test_report_name }}.xml'
rescue:

2
roles/tobiko-common/defaults/main.yaml
View File

@ -46,7 +46,7 @@ test_report_files:
- '{{ test_log_file | realpath }}'
# Test report files prefix
test_report_name: test_results
test_report_name: tobiko_results
# Log file where test cases should write logging messages to
test_log_file: '{{ test_report_dir | realpath }}/tobiko.log'

3
setup.cfg
View File

@ -49,3 +49,6 @@ tag_svn_revision = 0
[wheel]
universal = 1
[tool:pytest]
render_collapsed = True

14
test-requirements.txt
View File

@ -1,9 +1,9 @@
# Unit tests requirements
coverage>=4.5.0 # Apache-2.0
junitxml>=0.7.0 # LGPL-3
mock>=2.0.0 # BSD
os-testr>=1.0.0 # Apache-2.0
psutil>=5.7.2 # BSD
python-subunit>=1.4.0 # Apache-2.0
stestr>=3.0.0 # Apache-2.0
ansi2html>=1.6.0 # LGPLv3+
coverage>=4.5.0 # Apache-2.0
mock>=2.0.0 # BSD
psutil>=5.7.2 # BSD
pytest>=6.0.0 # MIT
pytest-html>=3.0.0 # MPL-2.0
pytest-xdist[psutil]>=2.0.0 # MIT

39
tobiko/tests/conftest.py Normal file
View File

@ -0,0 +1,39 @@
# Copyright (c) 2020 Red Hat
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
from datetime import datetime
from py.xml import html # pylint: disable=no-name-in-module,import-error
import pytest
def pytest_html_results_table_header(cells):
cells.insert(2, html.th("Description"))
cells.insert(1, html.th("Time", class_="sortable time", col="time"))
cells.pop()
def pytest_html_results_table_row(report, cells):
cells.insert(2, html.td(report.description))
cells.insert(1, html.td(datetime.utcnow(), class_="col-time"))
cells.pop()
@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call): # pylint: disable=unused-argument
outcome = yield
report = outcome.get_result()
report.description = str(item.function.__doc__)

62
tools/run_tests.py
View File

@ -29,26 +29,26 @@ from tools import common # noqa
LOG = common.get_logger(__name__)
OS_TEST_PATH = common.normalize_path(os.environ['OS_TEST_PATH'])
# Output dirs
TOX_REPORT_DIR = common.normalize_path(
os.environ.get('TOX_REPORT_DIR', os.getcwd()))
TOX_REPORT_NAME = os.environ.get('TOX_REPORT_NAME', 'test_results')
TOX_REPORT_NAME = os.environ.get('TOX_REPORT_NAME', 'tobiko_results')
TOX_REPORT_PREFIX = os.path.join(TOX_REPORT_DIR, TOX_REPORT_NAME)
TOX_REPORT_LOG = os.environ.get(
'TOX_REPORT_LOG', TOX_REPORT_PREFIX + '.log')
TOX_REPORT_SUBUNIT = os.environ.get(
'TOX_REPORT_SUBUNIT', TOX_REPORT_PREFIX + '.subunit')
TOX_REPORT_HTML = os.environ.get(
'TOX_REPORT_HTML', TOX_REPORT_PREFIX + '.html')
TOX_REPORT_XML = os.environ.get(
'TOX_REPORT_XML', TOX_REPORT_PREFIX + '.xml')
TOX_NUM_PROCESSES = os.environ.get('TOX_NUM_PROCESSES') or 'auto'
TOX_RUN_TESTS_TIMEOUT = float(os.environ.get('TOX_RUN_TESTS_TIMEOUT') or 0.)
TOX_PYDEV_DEBUG = bool(
@ -90,20 +90,6 @@ def run_tests():
LOG.error(f"Error while running test cases.\n{ex}")
succeeded = False
try:
log_tests_results()
except subprocess.CalledProcessError:
if succeeded:
raise
make_subunit_file()
make_html_file()
try:
make_xml_file()
except subprocess.CalledProcessError:
if succeeded:
raise
return succeeded
@ -137,8 +123,7 @@ def terminate_childs():
def cleanup_report_dir():
for report_file in [TOX_REPORT_LOG, TOX_REPORT_SUBUNIT, TOX_REPORT_HTML,
TOX_REPORT_XML]:
for report_file in [TOX_REPORT_LOG, TOX_REPORT_HTML, TOX_REPORT_XML]:
if not common.make_dir(os.path.dirname(report_file)):
common.remove_file(report_file)
@ -148,13 +133,6 @@ def log_environ():
capture_stdout=False)
def log_tests_results():
common.execute('stestr last --all-attachments >> "{log_file}"',
log_file=TOX_REPORT_LOG,
capture_stdout=False,
check=False)
def debug_test_cases():
common.execute_python('-m testtools.run {posargs}',
posargs=common.get_posargs(),
@ -162,28 +140,14 @@ def debug_test_cases():
def run_test_cases():
common.execute('stestr run {posargs}',
posargs=common.get_posargs(),
capture_stdout=False)
def make_subunit_file():
common.execute('stestr last --subunit > "{subunit_file}"',
subunit_file=TOX_REPORT_SUBUNIT,
capture_stdout=False)
def make_html_file():
common.execute('subunit2html "{subunit_file}" "{html_file}"',
subunit_file=TOX_REPORT_SUBUNIT,
html_file=TOX_REPORT_HTML,
capture_stdout=False)
def make_xml_file():
common.execute('subunit2junitxml "{subunit_file}" -o "{xml_file}"',
subunit_file=TOX_REPORT_SUBUNIT,
xml_file=TOX_REPORT_XML,
xdist_options = ''
if TOX_NUM_PROCESSES != '1':
xdist_options = f"--numprocesses {TOX_NUM_PROCESSES} --dist loadscope"
common.execute(f"pytest "
f"{xdist_options} "
f"--junitxml={TOX_REPORT_XML} "
f"--html={TOX_REPORT_HTML} --self-contained-html "
f"{common.get_posargs()}",
capture_stdout=False)

15
tox.ini
View File

@ -27,10 +27,11 @@ setenv =
OS_TEST_PATH = {toxinidir}/tobiko/tests/unit
PS1 = [tobiko@{envname}] {env:PS1:}
PYTHONWARNINGS = ignore::Warning,{env:PYTHONWARNINGS:}
RUN_TESTS_EXTRA_ARGS = --slowest
RUN_TESTS_EXTRA_ARGS = {env:OS_TEST_PATH}
TOBIKO_PREVENT_CREATE = {env:TOBIKO_PREVENT_CREATE:false}
TOX_REPORT_NAME = {env:TOX_REPORT_NAME:test_results}
TOX_REPORT_DIR = {env:TOX_REPORT_DIR:{toxinidir}/report/{envname}}
TOX_NUM_PROCESSES = {env:TOX_NUM_PROCESSES:auto}
TOX_REPORT_NAME = {env:TOX_REPORT_NAME:tobiko_results_{envname}}
TOX_REPORT_DIR = {env:TOX_REPORT_DIR:{envlogdir}}
TOX_CONSTRAINTS = {env:TOX_CONSTRAINTS:-c{env:TOX_CONSTRAINTS_FILE}}
TOX_CONSTRAINTS_FILE = {env:TOX_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt}
TOX_EXTRA_REQUIREMENTS = {env:TOX_EXTRA_REQUIREMENTS:-r{toxinidir}/extra-requirements.txt}
@ -52,7 +53,6 @@ envdir = {[testenv:py3]envdir}
setenv =
{[testenv]setenv}
PYTHON = coverage run --parallel-mode
RUN_TESTS_EXTRA_ARGS = --no-subunit-trace
TOX_COVER_DIR={env:TOX_COVER_DIR:{toxinidir}/cover}
commands =
find . -type f -name ".coverage.*" -delete
@ -61,7 +61,7 @@ commands =
coverage combine
coverage html -d "{env:TOX_COVER_DIR}"
coverage xml -o "{env:TOX_COVER_DIR}/cover/coverage.xml"
coverage report --fail-under=40 --skip-covered
coverage report --skip-covered
find . -type f -name ".coverage.*" -delete
whitelist_externals =
find
@ -209,7 +209,7 @@ passenv = {[integration]passenv}
setenv =
{[integration]setenv}
OS_TEST_PATH = {toxinidir}/tobiko/tests/faults
RUN_TESTS_EXTRA_ARGS = --serial
TOX_NUM_PROCESSES = 1
[testenv:instanceha]
@ -220,7 +220,8 @@ passenv = {[integration]passenv}
setenv =
{[integration]setenv}
OS_TEST_PATH = {toxinidir}/tobiko/tests/faults/iha
RUN_TESTS_EXTRA_ARGS = --serial
TOX_NUM_PROCESSES = 1
# --- CI workflow test environments -------------------------------------------