From 5af7c53c05541dd4190871e03f5a457f820bac6f Mon Sep 17 00:00:00 2001 From: Eduardo Olivares Date: Wed, 8 Nov 2023 18:23:27 +0100 Subject: [PATCH] Add support to skip tests from CLI With this patch, a new pytest optional arg has been added: --skipregex When it is used, tests are skipped when the regex is found in their test names. Fully qualified test names are used for this search. Example: tobiko.tests.unit.test_exception.TestException.test_init It can be used from CLI with pytest like this: $ .tox/py3/bin/pytest \ --skipregex='test_join_chunks_.*_bytes|test_join_chunks_with_unicodes' \ tobiko/tests/unit It can be used from CLI with tox like this: $ PYTEST_ADDOPTS="--skipregex='test_join_chunks_.*_bytes|test_join_chunks_with_unicodes'" \ TOX_PYTHON=python3.9 \ tox -e py3 It can be used from the Tobiko Infrared plugin by using the following option (limitation: the skipregex is applied to all the steps from the the executed Tobiko workflow): $ infrared tobiko ... --pytest-addopts "--skipregex='test_join_chunks_.*_bytes|test_join_chunks_with_unicodes'" It can be used from a zuul job by configuring the following variable within the zuul job yaml file (limitation: the skipregex is applied to all the steps from the executed Tobiko workflow): pytest_addopts_global: "--skipregex='test_join_chunks_.*_bytes|test_join_chunks_with_unicodes'" Change-Id: I8ee32ba467bd70142816953598d1736fa353d3d0 --- infrared_plugin/plugin.spec | 6 ++-- .../skipregex-new-arg-a9f1c001cbc823f9.yaml | 36 +++++++++++++++++++ roles/tobiko-tox/defaults/main.yaml | 3 +- roles/tobiko-tox/tasks/tox.yaml | 2 +- tobiko/tests/conftest.py | 19 ++++++++++ 5 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/skipregex-new-arg-a9f1c001cbc823f9.yaml diff --git a/infrared_plugin/plugin.spec b/infrared_plugin/plugin.spec index 12dcab7cd..7bc40f49d 100644 --- a/infrared_plugin/plugin.spec +++ b/infrared_plugin/plugin.spec @@ -202,8 +202,10 @@ subparsers: ansible_variable: tox_python pytest-addopts: type: Value - help: Extra options to be passed to PyTest - ansible_variable: pytest_addopts + help: > + Extra options to be passed to PyTest. This value is overriden + when it is defined at workflow level too. + ansible_variable: pytest_addopts_global pytest-markers: type: Value help: > diff --git a/releasenotes/notes/skipregex-new-arg-a9f1c001cbc823f9.yaml b/releasenotes/notes/skipregex-new-arg-a9f1c001cbc823f9.yaml new file mode 100644 index 000000000..104ee34fd --- /dev/null +++ b/releasenotes/notes/skipregex-new-arg-a9f1c001cbc823f9.yaml @@ -0,0 +1,36 @@ +--- +features: + - | + Added new input argument `--skipregex` + When it is used, tests are skipped when the regex is found in their test + names. Fully qualified test names are used for this search. + + | Example: tobiko.tests.unit.test_exception.TestException.test_init + + * It can be used from CLI with pytest + + .. code-block:: text + + $ .tox/py3/bin/pytest --skipregex='test_join_chunks_.*_bytes|test_join_chunks_with_unicodes' tobiko/tests/unit + + * It can be used from CLI with tox + + .. code-block:: text + + $ PYTEST_ADDOPTS="--skipregex='test_join_chunks_.*_bytes|test_join_chunks_with_unicodes'" TOX_PYTHON=python3.9 tox -e py3 + + * It can be used from the Tobiko Infrared plugin by using the --pytest-addopts + option (limitation: the skipregex is applied to all the steps from the + the executed Tobiko workflow) + + .. code-block:: text + + $ infrared tobiko ... --pytest-addopts "--skipregex='test_join_chunks_.*_bytes|test_join_chunks_with_unicodes'" + + * It can be used from a zuul job by configuring the following variable + within the zuul job yaml file (limitation: the skipregex is applied to + all the steps from the executed Tobiko workflow) + + .. code-block:: text + + pytest_addopts_global: "--skipregex='test_join_chunks_.*_bytes|test_join_chunks_with_unicodes'" diff --git a/roles/tobiko-tox/defaults/main.yaml b/roles/tobiko-tox/defaults/main.yaml index b04a5d427..e796c376d 100644 --- a/roles/tobiko-tox/defaults/main.yaml +++ b/roles/tobiko-tox/defaults/main.yaml @@ -36,7 +36,8 @@ tox_expected_rcs: - '{{ tox_succeeded_rc }}' ignore_test_failures: no -pytest_addopts: '' +pytest_addopts: '{{ pytest_addopts_global }}' +pytest_addopts_global: '' pytest_markers: "{% if not test_flaky%}not flaky{% endif %}" pytest_maxfail: '' test_flaky: false diff --git a/roles/tobiko-tox/tasks/tox.yaml b/roles/tobiko-tox/tasks/tox.yaml index 60b2a22e4..6cf1ae402 100644 --- a/roles/tobiko-tox/tasks/tox.yaml +++ b/roles/tobiko-tox/tasks/tox.yaml @@ -50,7 +50,7 @@ - name: reset values used to generate tox_constrain_env after executing a test suite set_fact: - pytest_addopts: '' + pytest_addopts: '{{ pytest_addopts_global }}' test_flaky: false pytest_markers: "{% if not test_flaky%}not flaky{% endif %}" pytest_maxfail: '' diff --git a/tobiko/tests/conftest.py b/tobiko/tests/conftest.py index 7507b688c..e2f031e7b 100644 --- a/tobiko/tests/conftest.py +++ b/tobiko/tests/conftest.py @@ -16,6 +16,7 @@ from __future__ import absolute_import from datetime import datetime import os +import re import subprocess from oslo_log import log @@ -172,3 +173,21 @@ def pytest_runtest_call(item): @pytest.fixture(scope="session", autouse=True) def cleanup_shelves(): tobiko.initialize_shelves() + + +def pytest_addoption(parser): + parser.addoption("--skipregex", action="store", + default="", help="skip tests matching the provided regex") + + +def pytest_collection_modifyitems(config, items): + skipregex = config.getoption("--skipregex") + if not skipregex: + # --skipregex not given in cli, therefore move on + return + skip_listed = pytest.mark.skip(reason="matches --skipregex") + for item in items: + fully_qualified_test_name = '.'.join([item.obj.__module__, + item.getmodpath()]) + if re.search(skipregex, fully_qualified_test_name): + item.add_marker(skip_listed)