tempest/tox.ini
melanie witt 4f9d735923 Use 'tempest run --slowest' in integrated-compute (again)
Change I53b0fa6dcf077191565e2aaf3f08a24b3238a34b added the --slowest
option to the tempest run command for scenario and serial tests but
missed adding it when running api tests.

The --slowest option will show a list of the slowest tests after the
tests finish running.

Change-Id: Ia5a995121748a2c244f060e6ce4b395213bb196e
2024-06-07 20:35:41 +00:00

501 lines
19 KiB
INI

[tox]
envlist = pep8,py39,bashate,pip-check-reqs
minversion = 3.18.0
[tempestenv]
sitepackages = False
setenv =
VIRTUAL_ENV={envdir}
OS_TEST_PATH=./tempest/test_discover
OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:1200}
deps =
-c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
-r{toxinidir}/requirements.txt
[testenv]
setenv =
VIRTUAL_ENV={envdir}
OS_LOG_CAPTURE=1
OS_STDOUT_CAPTURE=1
OS_STDERR_CAPTURE=1
OS_TEST_TIMEOUT=160
PYTHONWARNINGS=default::DeprecationWarning,ignore::DeprecationWarning:distutils,ignore::DeprecationWarning:site
passenv =
OS_STDOUT_CAPTURE
OS_STDERR_CAPTURE
OS_TEST_TIMEOUT
OS_TEST_LOCK_PATH
TEMPEST_CONFIG
TEMPEST_CONFIG_DIR
http_proxy
HTTP_PROXY
https_proxy
HTTPS_PROXY
no_proxy
NO_PROXY
ZUUL_CACHE_DIR
REQUIREMENTS_PIP_LOCATION
GENERATE_TEMPEST_PLUGIN_LIST
usedevelop = True
allowlist_externals =
find
deps =
-c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands =
find . -type f -name "*.pyc" -delete
stestr --test-path ./tempest/tests run {posargs}
[testenv:genconfig]
commands = oslo-config-generator --config-file tempest/cmd/config-generator.tempest.conf
[testenv:cover]
setenv =
{[testenv]setenv}
PYTHON=coverage run --source tempest --parallel-mode
commands =
coverage erase
find . -type f -name "*.pyc" -delete
stestr --test-path ./tempest/tests run {posargs}
coverage combine
coverage html -d cover
coverage xml -o cover/coverage.xml
coverage report -m --fail-under=76
[testenv:debug]
commands = oslo_debug_helper -t tempest/tests {posargs}
[testenv:all]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
# 'all' includes slow tests
setenv =
{[tempestenv]setenv}
deps = {[tempestenv]deps}
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex {posargs:''}
[testenv:all-plugin]
# DEPRECATED
# NOTE(andreaf) The all-plugin tox env uses sitepackages
# so that plugins installed outsite of Tempest virtual environment
# can be discovered. After the implementation during the Queens
# release cycle of the goal of moving Tempest plugins in dedicated
# git repos, this environment should not be used anymore. "all"
# should be used instead with the appropriate regex filtering.
sitepackages = True
# 'all' includes slow tests
setenv =
{[tempestenv]setenv}
deps = {[tempestenv]deps}
commands =
echo "WARNING: The all-plugin env is deprecated and will be removed"
echo "WARNING Please use the 'all' environment for Tempest plugins."
find . -type f -name "*.pyc" -delete
tempest run --regex {posargs:''}
[testenv:all-site-packages]
sitepackages = True
# 'all' includes slow tests
setenv =
{[tempestenv]setenv}
deps = {[tempestenv]deps}
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex {posargs:''}
[testenv:full]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag:
# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610
# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now.
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' {posargs}
tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)' {posargs}
[testenv:integrated-full]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run. It exclude the extra
# tests mentioned in tools/tempest-extra-tests-list.txt and slow tag:
# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610
# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now.
regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)'
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex {[testenv:integrated-full]regex1} --exclude-list ./tools/tempest-extra-tests-list.txt {posargs}
tempest run --combine --serial --regex {[testenv:integrated-full]regex2} {posargs}
[testenv:extra-tests]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select extra tests mentioned in
# tools/tempest-extra-tests-list.txt and exclude slow tag tests:
# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610
# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now.
exclude-regex = '\[.*\bslow\b.*\]'
commands =
find . -type f -name "*.pyc" -delete
tempest run --exclude-regex {[testenv:extra-tests]exclude-regex} --include-list ./tools/tempest-extra-tests-list.txt {posargs}
[testenv:full-parallel]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# But exclude the extra tests mentioned in tools/tempest-extra-tests-list.txt
regex = '(^tempest\.scenario.*)|(^tempest\.serial_tests)|(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex {[testenv:full-parallel]regex} --exclude-list ./tools/tempest-extra-tests-list.txt {posargs}
[testenv:api-microversion-tests]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
regex = '(^tempest\.api\.compute)|(^tempest\.api\.volume)'
# The regex below is used to select all tempest api tests for services having API
# microversion concept.
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex {[testenv:api-microversion-tests]regex} {posargs}
[testenv:integrated-network]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)'
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in exclude-list file:
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex {[testenv:integrated-network]regex1} --exclude-list ./tools/tempest-integrated-gate-networking-exclude-list.txt {posargs}
tempest run --combine --serial --regex {[testenv:integrated-network]regex2} --exclude-list ./tools/tempest-integrated-gate-networking-exclude-list.txt {posargs}
[testenv:integrated-compute]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)'
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in exclude-list file:
commands =
find . -type f -name "*.pyc" -delete
tempest run --slowest --regex {[testenv:integrated-compute]regex1} --exclude-list ./tools/tempest-integrated-gate-compute-exclude-list.txt {posargs}
tempest run --combine --serial --slowest --regex {[testenv:integrated-compute]regex2} --exclude-list ./tools/tempest-integrated-gate-compute-exclude-list.txt {posargs}
[testenv:integrated-placement]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)'
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in exclude-list file:
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex {[testenv:integrated-placement]regex1} --exclude-list ./tools/tempest-integrated-gate-placement-exclude-list.txt {posargs}
tempest run --combine --serial --regex {[testenv:integrated-placement]regex2} --exclude-list ./tools/tempest-integrated-gate-placement-exclude-list.txt {posargs}
[testenv:integrated-storage]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)'
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in exclude-list file:
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex {[testenv:integrated-storage]regex1} --exclude-list ./tools/tempest-integrated-gate-storage-exclude-list.txt {posargs}
tempest run --combine --serial --regex {[testenv:integrated-storage]regex2} --exclude-list ./tools/tempest-integrated-gate-storage-exclude-list.txt {posargs}
[testenv:integrated-object-storage]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)'
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in exclude-list file:
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex {[testenv:integrated-object-storage]regex1} --exclude-list ./tools/tempest-integrated-gate-object-storage-exclude-list.txt {posargs}
tempest run --combine --serial --regex {[testenv:integrated-object-storage]regex2} --exclude-list ./tools/tempest-integrated-gate-object-storage-exclude-list.txt {posargs}
[testenv:full-serial]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
regex = '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|serial_tests))'
# The regex below is used to select which tests to run and exclude the slow tag:
# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610
# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now.
commands =
find . -type f -name "*.pyc" -delete
tempest run --serial --regex {[testenv:full-serial]regex} {posargs}
[testenv:scenario]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
regex = '(^tempest\.scenario)'
# The regex below is used to select all scenario tests
commands =
find . -type f -name "*.pyc" -delete
tempest run --serial --regex {[testenv:scenario]regex} {posargs}
[testenv:smoke]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
regex = '\[.*\bsmoke\b.*\]'
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex {[testenv:smoke]regex} {posargs}
[testenv:smoke-serial]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
regex = '\[.*\bsmoke\b.*\]'
# This is still serial because neutron doesn't work with parallel. See:
# https://bugs.launchpad.net/tempest/+bug/1216076 so the neutron smoke
# job would fail if we moved it to parallel.
commands =
find . -type f -name "*.pyc" -delete
tempest run --serial --regex {[testenv:smoke-serial]regex} {posargs}
[testenv:slow-serial]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
regex = '\[.*\bslow\b.*\]'
# The regex below is used to select the slow tagged tests to run serially:
commands =
find . -type f -name "*.pyc" -delete
tempest run --serial --regex {[testenv:slow-serial]regex} {posargs}
[testenv:slow]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select the slow tagged tests:
regex = '\[.*\bslow\b.*\]'
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex {[testenv:slow]regex} {posargs}
[testenv:multinode]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select the multinode and smoke tagged tests
regex = '\[.*\bsmoke|multinode\b.*\]'
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex {[testenv:multinode]regex} {posargs}
[testenv:ipv6-only]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
regex = '\[.*\bsmoke|ipv6|test_network_v6\b.*\]'
# Run only smoke and ipv6 tests. This env is used to tests
# the ipv6 deployments and basic tests run fine so that we can
# verify that services listen on IPv6 address.
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex {[testenv:ipv6-only]regex} {posargs}
[testenv:venv]
deps =
-c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
-r{toxinidir}/requirements.txt
-r{toxinidir}/doc/requirements.txt
commands = {posargs}
[testenv:venv-tempest]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
commands = {posargs}
[testenv:docs]
deps =
-c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
-r{toxinidir}/doc/requirements.txt
commands =
sphinx-apidoc -f -o doc/source/tests/compute tempest/api/compute
sphinx-apidoc -f -o doc/source/tests/identity tempest/api/identity
sphinx-apidoc -f -o doc/source/tests/image tempest/api/image
sphinx-apidoc -f -o doc/source/tests/network tempest/api/network
sphinx-apidoc -f -o doc/source/tests/object_storage tempest/api/object_storage
sphinx-apidoc -f -o doc/source/tests/volume tempest/api/volume
sphinx-apidoc -f -o doc/source/tests/scenario tempest/scenario
sphinx-apidoc -f -o doc/source/tests/serial_tests tempest/serial_tests
rm -rf doc/build
sphinx-build -W -b html doc/source doc/build/html
allowlist_externals =
rm
[testenv:pdf-docs]
deps = {[testenv:docs]deps}
allowlist_externals =
rm
make
commands =
sphinx-apidoc -f -o doc/source/tests/compute tempest/api/compute
sphinx-apidoc -f -o doc/source/tests/identity tempest/api/identity
sphinx-apidoc -f -o doc/source/tests/image tempest/api/image
sphinx-apidoc -f -o doc/source/tests/network tempest/api/network
sphinx-apidoc -f -o doc/source/tests/object_storage tempest/api/object_storage
sphinx-apidoc -f -o doc/source/tests/volume tempest/api/volume
sphinx-apidoc -f -o doc/source/tests/scenario tempest/scenario
sphinx-apidoc -f -o doc/source/tests/serial_tests tempest/serial_tests
sphinx-build -W -b latex doc/source doc/build/pdf
make -C doc/build/pdf
[testenv:pep8]
deps =
{[testenv]deps}
autopep8
commands =
autopep8 --exit-code --max-line-length=79 --experimental --diff -r tempest setup.py
flake8 {posargs}
check-uuid
[testenv:autopep8]
deps = autopep8
commands =
{toxinidir}/tools/format.sh
[testenv:uuidgen]
commands =
check-uuid --fix
[hacking]
import_exceptions = tempest.services
[flake8]
# E125 is a won't fix until https://github.com/jcrocholl/pep8/issues/126 is resolved. For further detail see https://review.opendev.org/#/c/36788/
# E123 skipped because it is ignored by default in the default pep8
# E129 skipped because it is too limiting when combined with other rules
# W504 skipped because it is overeager and unnecessary
# H405 skipped because it arbitrarily forces doctring "title" lines
ignore = E125,E123,E129,W504,H405
show-source = True
exclude = .git,.venv,.tox,dist,doc,*egg,build
enable-extensions = H106,H203,H904
import-order-style = pep8
[flake8:local-plugins]
extension =
T102 = checks:import_no_clients_in_api_and_scenario_tests
T104 = checks:scenario_tests_need_service_tags
T105 = checks:no_setup_teardown_class_for_tests
T107 = checks:service_tags_not_in_module_path
T108 = checks:no_hyphen_at_end_of_rand_name
N322 = checks:no_mutable_default_args
T109 = checks:no_testtools_skip_decorator
T110 = checks:get_resources_on_service_clients
T111 = checks:delete_resources_on_service_clients
T112 = checks:dont_import_local_tempest_into_lib
T113 = checks:use_rand_uuid_instead_of_uuid4
T114 = checks:dont_use_config_in_tempest_lib
T115 = checks:dont_put_admin_tests_on_nonadmin_path
T116 = checks:unsupported_exception_attribute_PY3
T117 = checks:negative_test_attribute_always_applied_to_negative_tests
T118 = checks:no_log_warn
paths =
./tempest/hacking
[testenv:releasenotes]
deps =
-c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
-r{toxinidir}/doc/requirements.txt
commands =
rm -rf releasenotes/build
sphinx-build -a -E -W -d releasenotes/build/doctrees \
-b html releasenotes/source releasenotes/build/html
allowlist_externals = rm
[testenv:bashate]
# if you want to test out some changes you have made to bashate
# against tempest, just set BASHATE_INSTALL_PATH=/path/... to your
# modified bashate tree
deps =
{env:BASHATE_INSTALL_PATH:bashate}
allowlist_externals = bash
commands = bash -c "find {toxinidir}/tools \
-not \( -type d -name .?\* -prune \) \
-type f \
-name \*.sh \
-print0 | xargs -0 bashate -v -eE005,E042 -i E006"
[testenv:pip-check-reqs]
# Do not install test-requirements as that will pollute the virtualenv for
# determining missing packages.
# This also means that pip-check-reqs must be installed separately, outside
# of the requirements.txt files
deps = pip_check_reqs
-r{toxinidir}/requirements.txt
commands=
pip-extra-reqs -d --ignore-file=tempest/tests/* tempest
pip-missing-reqs -d --ignore-file=tempest/tests/* tempest
[testenv:bindep]
# Do not install any requirements. We want this to be fast and work even if
# system dependencies are missing, since it's used to tell you what system
# dependencies are missing! This also means that bindep must be installed
# separately, outside of the requirements files.
deps = bindep
commands = bindep test
[testenv:plugin-sanity-check]
# perform tempest plugin sanity
allowlist_externals = bash
commands =
bash tools/tempest-plugin-sanity.sh
[testenv:stestr-master]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
regex = '\[.*\bsmoke\b.*\]'
# The below command install stestr master version and run smoke tests
commands =
find . -type f -name "*.pyc" -delete
pip install -U git+https://github.com/mtreinish/stestr
tempest run --regex {[testenv:stestr-master]regex} {posargs}