tempest/tox.ini

[tox]
envlist = pep8,py36,py39,bashate,pip-check-reqs
minversion = 3.18.0
skipsdist = True
ignore_basepython_conflict = True

[tempestenv]
basepython = python3
sitepackages = False
setenv =
    VIRTUAL_ENV={envdir}
    OS_TEST_PATH=./tempest/test_discover
    OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:1200}
deps =
    -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
    -r{toxinidir}/requirements.txt

[testenv]
basepython = python3
setenv =
    VIRTUAL_ENV={envdir}
    OS_LOG_CAPTURE=1
    OS_STDOUT_CAPTURE=1
    OS_STDERR_CAPTURE=1
    OS_TEST_TIMEOUT=160
    PYTHONWARNINGS=default::DeprecationWarning,ignore::DeprecationWarning:distutils,ignore::DeprecationWarning:site
passenv = OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_TEST_TIMEOUT OS_TEST_LOCK_PATH TEMPEST_CONFIG TEMPEST_CONFIG_DIR http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY ZUUL_CACHE_DIR REQUIREMENTS_PIP_LOCATION GENERATE_TEMPEST_PLUGIN_LIST
usedevelop = True
install_command = pip install {opts} {packages}
allowlist_externals = *
deps =
    -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
    -r{toxinidir}/requirements.txt
    -r{toxinidir}/test-requirements.txt
commands =
    find . -type f -name "*.pyc" -delete
    stestr --test-path ./tempest/tests run {posargs}

[testenv:genconfig]
commands = oslo-config-generator --config-file tempest/cmd/config-generator.tempest.conf

[testenv:cover]
setenv =
  {[testenv]setenv}
  PYTHON=coverage run --source tempest --parallel-mode
commands =
  coverage erase
  find . -type f -name "*.pyc" -delete
  stestr --test-path ./tempest/tests run {posargs}
  coverage combine
  coverage html -d cover
  coverage xml -o cover/coverage.xml
  coverage report

[testenv:debug]
commands = oslo_debug_helper -t tempest/tests {posargs}

[testenv:all]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
# 'all' includes slow tests
setenv =
    {[tempestenv]setenv}
deps = {[tempestenv]deps}
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --regex {posargs:''}

[testenv:all-plugin]
# DEPRECATED
# NOTE(andreaf) The all-plugin tox env uses sitepackages
# so that plugins installed outsite of Tempest virtual environment
# can be discovered. After the implementation during the Queens
# release cycle of the goal of moving Tempest plugins in dedicated
# git repos, this environment should not be used anymore. "all"
# should be used instead with the appropriate regex filtering.
sitepackages = True
# 'all' includes slow tests
setenv =
    {[tempestenv]setenv}
basepython = {[tempestenv]basepython}
deps = {[tempestenv]deps}
commands =
    echo "WARNING: The all-plugin env is deprecated and will be removed"
    echo "WARNING  Please use the 'all' environment for Tempest plugins."
    find . -type f -name "*.pyc" -delete
    tempest run --regex {posargs:''}

[testenv:all-site-packages]
sitepackages = True
# 'all' includes slow tests
setenv =
    {[tempestenv]setenv}
basepython = {[tempestenv]basepython}
deps = {[tempestenv]deps}
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --regex {posargs:''}

[testenv:full]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag:
# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610
# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now.
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' {posargs}
    tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' {posargs}

[testenv:full-parallel]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select all tempest scenario and including the non slow api tests
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --regex '(^tempest\.scenario.*)|(?!.*\[.*\bslow\b.*\])(^tempest\.api)' {posargs}

[testenv:api-microversion-tests]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select all tempest api tests for services having API
# microversion concept.
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --regex '(^tempest\.api\.compute)|(^tempest\.api\.volume)' {posargs}

[testenv:integrated-network]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in exclude-list file:
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-networking-exclude-list.txt {posargs}
    tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exclude-list ./tools/tempest-integrated-gate-networking-exclude-list.txt {posargs}

[testenv:integrated-compute]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in exclude-list file:
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-compute-exclude-list.txt {posargs}
    tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exclude-list ./tools/tempest-integrated-gate-compute-exclude-list.txt {posargs}

[testenv:integrated-placement]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in exclude-list file:
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-placement-exclude-list.txt {posargs}
    tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exclude-list ./tools/tempest-integrated-gate-placement-exclude-list.txt {posargs}

[testenv:integrated-storage]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in exclude-list file:
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-storage-exclude-list.txt {posargs}
    tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exclude-list ./tools/tempest-integrated-gate-storage-exclude-list.txt {posargs}

[testenv:integrated-object-storage]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in exclude-list file:
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-object-storage-exclude-list.txt {posargs}
    tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exclude-list ./tools/tempest-integrated-gate-object-storage-exclude-list.txt {posargs}

[testenv:full-serial]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag:
# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610
# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now.
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario))' {posargs}

[testenv:scenario]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select all scenario tests
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --serial --regex '(^tempest\.scenario)' {posargs}

[testenv:smoke]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --regex '\[.*\bsmoke\b.*\]' {posargs}

[testenv:smoke-serial]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# This is still serial because neutron doesn't work with parallel. See:
# https://bugs.launchpad.net/tempest/+bug/1216076 so the neutron smoke
# job would fail if we moved it to parallel.
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --serial --regex '\[.*\bsmoke\b.*\]' {posargs}

[testenv:slow-serial]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select the slow tagged tests to run serially:
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --serial --regex '\[.*\bslow\b.*\]' {posargs}

[testenv:ipv6-only]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# Run only smoke and ipv6 tests. This env is used to tests
# the ipv6 deployments and basic tests run fine so that we can
# verify that services listen on IPv6 address.
commands =
    find . -type f -name "*.pyc" -delete
    tempest run --regex '\[.*\bsmoke|ipv6|test_network_v6\b.*\]' {posargs}

[testenv:venv]
deps =
  -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
  -r{toxinidir}/requirements.txt
  -r{toxinidir}/doc/requirements.txt
commands = {posargs}

[testenv:venv-tempest]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
commands = {posargs}

[testenv:docs]
deps =
  -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
  -r{toxinidir}/doc/requirements.txt
commands =
  sphinx-apidoc -f -o doc/source/tests/compute tempest/api/compute
  sphinx-apidoc -f -o doc/source/tests/identity tempest/api/identity
  sphinx-apidoc -f -o doc/source/tests/image tempest/api/image
  sphinx-apidoc -f -o doc/source/tests/network tempest/api/network
  sphinx-apidoc -f -o doc/source/tests/object_storage tempest/api/object_storage
  sphinx-apidoc -f -o doc/source/tests/scenario tempest/scenario
  sphinx-apidoc -f -o doc/source/tests/volume tempest/api/volume
  rm -rf doc/build
  sphinx-build -W -b html doc/source doc/build/html
allowlist_externals =
    rm

[testenv:pdf-docs]
deps = {[testenv:docs]deps}
allowlist_externals =
   rm
   make
commands =
   sphinx-apidoc -f -o doc/source/tests/compute tempest/api/compute
   sphinx-apidoc -f -o doc/source/tests/identity tempest/api/identity
   sphinx-apidoc -f -o doc/source/tests/image tempest/api/image
   sphinx-apidoc -f -o doc/source/tests/network tempest/api/network
   sphinx-apidoc -f -o doc/source/tests/object_storage tempest/api/object_storage
   sphinx-apidoc -f -o doc/source/tests/scenario tempest/scenario
   sphinx-apidoc -f -o doc/source/tests/volume tempest/api/volume
   sphinx-build -W -b latex doc/source doc/build/pdf
   make -C doc/build/pdf

[testenv:pep8]
deps =
    -r{toxinidir}/test-requirements.txt
    autopep8
commands =
    autopep8 --exit-code --max-line-length=79 --experimental --diff -r tempest setup.py
    flake8 {posargs}
    check-uuid

[testenv:autopep8]
deps = autopep8
commands =
    {toxinidir}/tools/format.sh

[testenv:uuidgen]
commands =
    check-uuid --fix

[hacking]
import_exceptions = tempest.services

[flake8]
# E125 is a won't fix until https://github.com/jcrocholl/pep8/issues/126 is resolved.  For further detail see https://review.opendev.org/#/c/36788/
# E123 skipped because it is ignored by default in the default pep8
# E129 skipped because it is too limiting when combined with other rules
# W504 skipped because it is overeager and unnecessary
# H405 skipped because it arbitrarily forces doctring "title" lines
ignore = E125,E123,E129,W504,H405
show-source = True
exclude = .git,.venv,.tox,dist,doc,*egg,build
enable-extensions = H106,H203,H904
import-order-style = pep8

[flake8:local-plugins]
extension =
  T102 = checks:import_no_clients_in_api_and_scenario_tests
  T104 = checks:scenario_tests_need_service_tags
  T105 = checks:no_setup_teardown_class_for_tests
  T107 = checks:service_tags_not_in_module_path
  T108 = checks:no_hyphen_at_end_of_rand_name
  N322 = checks:no_mutable_default_args
  T109 = checks:no_testtools_skip_decorator
  T110 = checks:get_resources_on_service_clients
  T111 = checks:delete_resources_on_service_clients
  T112 = checks:dont_import_local_tempest_into_lib
  T113 = checks:use_rand_uuid_instead_of_uuid4
  T114 = checks:dont_use_config_in_tempest_lib
  T115 = checks:dont_put_admin_tests_on_nonadmin_path
  T116 = checks:unsupported_exception_attribute_PY3
  T117 = checks:negative_test_attribute_always_applied_to_negative_tests
paths =
  ./tempest/hacking

[testenv:releasenotes]
deps =
  -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
  -r{toxinidir}/doc/requirements.txt
commands =
  rm -rf releasenotes/build
  sphinx-build -a -E -W -d releasenotes/build/doctrees \
         -b html releasenotes/source releasenotes/build/html
allowlist_externals = rm

[testenv:bashate]
# if you want to test out some changes you have made to bashate
# against tempest, just set BASHATE_INSTALL_PATH=/path/... to your
# modified bashate tree
deps =
   {env:BASHATE_INSTALL_PATH:bashate}
allowlist_externals = bash
commands = bash -c "find {toxinidir}/tools    \
         -not \( -type d -name .?\* -prune \) \
         -type f                              \
         -name \*.sh                          \
         -print0 | xargs -0 bashate -v -eE005,E042 -i E006"

[testenv:pip-check-reqs]
# Do not install test-requirements as that will pollute the virtualenv for
# determining missing packages.
# This also means that pip-check-reqs must be installed separately, outside
# of the requirements.txt files
deps = pip_check_reqs
       -r{toxinidir}/requirements.txt
commands=
    pip-extra-reqs -d --ignore-file=tempest/tests/* tempest
    pip-missing-reqs -d --ignore-file=tempest/tests/* tempest


[testenv:bindep]
# Do not install any requirements. We want this to be fast and work even if
# system dependencies are missing, since it's used to tell you what system
# dependencies are missing! This also means that bindep must be installed
# separately, outside of the requirements files.
deps = bindep
commands = bindep test

[testenv:plugin-sanity-check]
# perform tempest plugin sanity
allowlist_externals = bash
commands =
  bash tools/tempest-plugin-sanity.sh

[testenv:stestr-master]
envdir = .tox/tempest
sitepackages = {[tempestenv]sitepackages}
basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The below command install stestr master version and run smoke tests
commands =
    find . -type f -name "*.pyc" -delete
    pip install -U git+https://github.com/mtreinish/stestr
    tempest run --regex '\[.*\bsmoke\b.*\]' {posargs}