diff --git a/tools/tempest-extra-tests-list.txt b/tools/tempest-extra-tests-list.txt new file mode 100644 index 0000000000..9c8810909a --- /dev/null +++ b/tools/tempest-extra-tests-list.txt @@ -0,0 +1,20 @@ +# This file includes the list of tests which need to be +# excluded to run from integrated testing (tempest-full job +# or other generic jobs. We will run these tests in a separate +# jobs. This is needed to avoid the job timeout, details in +# bug#2004780. +# Basic criteria to add test in this list is: +# * Admin test which are not needed for interop and most of them +# are running as part of other API and Scenario tests. +# * Negative tests which are mostly covered in tempest API tests +# or service unit/functional tests. + +# All admin tests except keystone admin test which might not have much +# coverage in existing other tests +tempest.api.compute.admin +tempest.api.volume.admin +tempest.api.image.admin +tempest.api.network.admin + +# All negative tests +negative diff --git a/tox.ini b/tox.ini index 972c05e782..47ef5ebc29 100644 --- a/tox.ini +++ b/tox.ini @@ -126,17 +126,49 @@ commands = tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' {posargs} tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)' {posargs} +[testenv:integrated-full] +envdir = .tox/tempest +sitepackages = {[tempestenv]sitepackages} +basepython = {[tempestenv]basepython} +setenv = {[tempestenv]setenv} +deps = {[tempestenv]deps} +# The regex below is used to select which tests to run. It exclude the extra +# tests mentioned in tools/tempest-extra-tests-list.txt and slow tag: +# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610 +# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now. +regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' +regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)' +commands = + find . -type f -name "*.pyc" -delete + tempest run --regex {[testenv:integrated-full]regex1} --exclude-list ./tools/tempest-extra-tests-list.txt {posargs} + tempest run --combine --serial --regex {[testenv:integrated-full]regex2} {posargs} + +[testenv:extra-tests] +envdir = .tox/tempest +sitepackages = {[tempestenv]sitepackages} +basepython = {[tempestenv]basepython} +setenv = {[tempestenv]setenv} +deps = {[tempestenv]deps} +# The regex below is used to select extra tests mentioned in +# tools/tempest-extra-tests-list.txt and exclude slow tag tests: +# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610 +# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now. +exclude-regex = '\[.*\bslow\b.*\]' +commands = + find . -type f -name "*.pyc" -delete + tempest run --exclude-regex {[testenv:extra-tests]exclude-regex} --include-list ./tools/tempest-extra-tests-list.txt {posargs} + [testenv:full-parallel] envdir = .tox/tempest sitepackages = {[tempestenv]sitepackages} basepython = {[tempestenv]basepython} setenv = {[tempestenv]setenv} deps = {[tempestenv]deps} +# But exlcude the extra tests mentioned in tools/tempest-extra-tests-list.txt regex = '(^tempest\.scenario.*)|(^tempest\.serial_tests)|(?!.*\[.*\bslow\b.*\])(^tempest\.api)' -# The regex below is used to select all tempest scenario and including the non slow api tests commands = find . -type f -name "*.pyc" -delete - tempest run --regex {[testenv:full-parallel]regex} {posargs} + tempest run --regex {[testenv:full-parallel]regex} --exclude-list ./tools/tempest-extra-tests-list.txt {posargs} [testenv:api-microversion-tests] envdir = .tox/tempest diff --git a/zuul.d/integrated-gate.yaml b/zuul.d/integrated-gate.yaml index a67f6b4985..d88c988946 100644 --- a/zuul.d/integrated-gate.yaml +++ b/zuul.d/integrated-gate.yaml @@ -59,6 +59,15 @@ # tests fail at a high rate (see bugs 1483434, 1813217, 1745168) c-bak: false +- job: + name: tempest-extra-tests + parent: devstack-tempest + description: | + This job runs the extra tests mentioned in + tools/tempest-extra-tests-list.txt. + vars: + tox_envlist: extra-tests + - job: name: tempest-full-py3 parent: devstack-tempest @@ -74,7 +83,7 @@ required-projects: - openstack/horizon vars: - tox_envlist: full + tox_envlist: integrated-full devstack_localrc: USE_PYTHON3: true FORCE_CONFIG_DRIVE: true @@ -107,6 +116,7 @@ # Required until bug/1949606 is resolved when using libvirt and QEMU # >=5.0.0 with a [libvirt]virt_type of qemu (TCG). configure_swap_size: 4096 + tox_envlist: full - job: name: tempest-integrated-networking diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 126119bce2..4281d6b71e 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -26,6 +26,8 @@ - ^.gitignore$ - ^.gitreview$ - ^.mailmap$ + - tempest-extra-tests: + irrelevant-files: *tempest-irrelevant-files - tempest-full-ubuntu-focal: irrelevant-files: *tempest-irrelevant-files - glance-multistore-cinder-import: @@ -63,6 +65,7 @@ - ^tools/tempest-integrated-gate-placement-exclude-list.txt - ^tools/tempest-integrated-gate-storage-blacklist.txt - ^tools/tempest-integrated-gate-storage-exclude-list.txt + - ^tools/tempest-extra-tests-list.txt - ^tools/verify-ipv6-only-deployments.sh - ^tools/with_venv.sh # tools/ is not here since this relies on a script in tools/. @@ -86,6 +89,7 @@ - ^tools/tempest-integrated-gate-placement-exclude-list.txt - ^tools/tempest-integrated-gate-storage-blacklist.txt - ^tools/tempest-integrated-gate-storage-exclude-list.txt + - ^tools/tempest-extra-tests-list.txt - ^tools/tempest-plugin-sanity.sh - ^tools/with_venv.sh - ^.coveragerc$ @@ -131,6 +135,8 @@ irrelevant-files: *tempest-irrelevant-files - tempest-full-py3: irrelevant-files: *tempest-irrelevant-files + - tempest-extra-tests: + irrelevant-files: *tempest-irrelevant-files - grenade: irrelevant-files: *tempest-irrelevant-files - tempest-ipv6-only: @@ -177,6 +183,9 @@ - tempest-slow-zed - tempest-slow-yoga - tempest-slow-xena + - tempest-full-zed-extra-tests + - tempest-full-yoga-extra-tests + - tempest-full-xena-extra-tests periodic: jobs: - tempest-all diff --git a/zuul.d/stable-jobs.yaml b/zuul.d/stable-jobs.yaml index fb2300b252..f70e79cce5 100644 --- a/zuul.d/stable-jobs.yaml +++ b/zuul.d/stable-jobs.yaml @@ -17,6 +17,24 @@ nodeset: openstack-single-node-focal override-checkout: stable/xena +- job: + name: tempest-full-zed-extra-tests + parent: tempest-extra-tests + nodeset: openstack-single-node-focal + override-checkout: stable/zed + +- job: + name: tempest-full-yoga-extra-tests + parent: tempest-extra-tests + nodeset: openstack-single-node-focal + override-checkout: stable/yoga + +- job: + name: tempest-full-xena-extra-tests + parent: tempest-extra-tests + nodeset: openstack-single-node-focal + override-checkout: stable/xena + - job: name: tempest-slow-zed parent: tempest-slow-py3 diff --git a/zuul.d/tempest-specific.yaml b/zuul.d/tempest-specific.yaml index 684270a5d9..972123ee95 100644 --- a/zuul.d/tempest-specific.yaml +++ b/zuul.d/tempest-specific.yaml @@ -30,6 +30,8 @@ - opendev.org/openstack/oslo.utils - opendev.org/openstack/oslo.versionedobjects - opendev.org/openstack/oslo.vmware + vars: + tox_envlist: full - job: name: tempest-full-parallel