From 518e426ab4ff28db11654f8309241ab215b0e42b Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Fri, 10 Feb 2023 19:57:36 -0600 Subject: [PATCH] Separate the extra tests to run in a separate job Recently we are seeing a lot of job timeout(bug#2004780) and we see many tests taking time and also number of tests increasing over time. This is to prepare the list of extra tests (here extra tests means the tests which are covered by the other API or scenario tests) which we do not need to run in every integrated jobs. Instead, we can run them in a separete job(s). Currently I am adding admin (except keystone) and negative tests in the 'extra tests' list but we can add more tests here which we think are covered in some other tests. As negative tests are important for interop, adding those extra tests coverage for stable branch job also but running them in periodic run only. Related-Bug: #2004780 Change-Id: Id02221df0d6180519751c63e890851bd59fdafa0 --- tools/tempest-extra-tests-list.txt | 20 +++++++++++++++++ tox.ini | 36 ++++++++++++++++++++++++++++-- zuul.d/integrated-gate.yaml | 12 +++++++++- zuul.d/project.yaml | 9 ++++++++ zuul.d/stable-jobs.yaml | 18 +++++++++++++++ zuul.d/tempest-specific.yaml | 2 ++ 6 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 tools/tempest-extra-tests-list.txt diff --git a/tools/tempest-extra-tests-list.txt b/tools/tempest-extra-tests-list.txt new file mode 100644 index 0000000000..9c8810909a --- /dev/null +++ b/tools/tempest-extra-tests-list.txt @@ -0,0 +1,20 @@ +# This file includes the list of tests which need to be +# excluded to run from integrated testing (tempest-full job +# or other generic jobs. We will run these tests in a separate +# jobs. This is needed to avoid the job timeout, details in +# bug#2004780. +# Basic criteria to add test in this list is: +# * Admin test which are not needed for interop and most of them +# are running as part of other API and Scenario tests. +# * Negative tests which are mostly covered in tempest API tests +# or service unit/functional tests. + +# All admin tests except keystone admin test which might not have much +# coverage in existing other tests +tempest.api.compute.admin +tempest.api.volume.admin +tempest.api.image.admin +tempest.api.network.admin + +# All negative tests +negative diff --git a/tox.ini b/tox.ini index 972c05e782..47ef5ebc29 100644 --- a/tox.ini +++ b/tox.ini @@ -126,17 +126,49 @@ commands = tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' {posargs} tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)' {posargs} +[testenv:integrated-full] +envdir = .tox/tempest +sitepackages = {[tempestenv]sitepackages} +basepython = {[tempestenv]basepython} +setenv = {[tempestenv]setenv} +deps = {[tempestenv]deps} +# The regex below is used to select which tests to run. It exclude the extra +# tests mentioned in tools/tempest-extra-tests-list.txt and slow tag: +# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610 +# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now. +regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' +regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)' +commands = + find . -type f -name "*.pyc" -delete + tempest run --regex {[testenv:integrated-full]regex1} --exclude-list ./tools/tempest-extra-tests-list.txt {posargs} + tempest run --combine --serial --regex {[testenv:integrated-full]regex2} {posargs} + +[testenv:extra-tests] +envdir = .tox/tempest +sitepackages = {[tempestenv]sitepackages} +basepython = {[tempestenv]basepython} +setenv = {[tempestenv]setenv} +deps = {[tempestenv]deps} +# The regex below is used to select extra tests mentioned in +# tools/tempest-extra-tests-list.txt and exclude slow tag tests: +# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610 +# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now. +exclude-regex = '\[.*\bslow\b.*\]' +commands = + find . -type f -name "*.pyc" -delete + tempest run --exclude-regex {[testenv:extra-tests]exclude-regex} --include-list ./tools/tempest-extra-tests-list.txt {posargs} + [testenv:full-parallel] envdir = .tox/tempest sitepackages = {[tempestenv]sitepackages} basepython = {[tempestenv]basepython} setenv = {[tempestenv]setenv} deps = {[tempestenv]deps} +# But exlcude the extra tests mentioned in tools/tempest-extra-tests-list.txt regex = '(^tempest\.scenario.*)|(^tempest\.serial_tests)|(?!.*\[.*\bslow\b.*\])(^tempest\.api)' -# The regex below is used to select all tempest scenario and including the non slow api tests commands = find . -type f -name "*.pyc" -delete - tempest run --regex {[testenv:full-parallel]regex} {posargs} + tempest run --regex {[testenv:full-parallel]regex} --exclude-list ./tools/tempest-extra-tests-list.txt {posargs} [testenv:api-microversion-tests] envdir = .tox/tempest diff --git a/zuul.d/integrated-gate.yaml b/zuul.d/integrated-gate.yaml index a67f6b4985..d88c988946 100644 --- a/zuul.d/integrated-gate.yaml +++ b/zuul.d/integrated-gate.yaml @@ -59,6 +59,15 @@ # tests fail at a high rate (see bugs 1483434, 1813217, 1745168) c-bak: false +- job: + name: tempest-extra-tests + parent: devstack-tempest + description: | + This job runs the extra tests mentioned in + tools/tempest-extra-tests-list.txt. + vars: + tox_envlist: extra-tests + - job: name: tempest-full-py3 parent: devstack-tempest @@ -74,7 +83,7 @@ required-projects: - openstack/horizon vars: - tox_envlist: full + tox_envlist: integrated-full devstack_localrc: USE_PYTHON3: true FORCE_CONFIG_DRIVE: true @@ -107,6 +116,7 @@ # Required until bug/1949606 is resolved when using libvirt and QEMU # >=5.0.0 with a [libvirt]virt_type of qemu (TCG). configure_swap_size: 4096 + tox_envlist: full - job: name: tempest-integrated-networking diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 126119bce2..4281d6b71e 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -26,6 +26,8 @@ - ^.gitignore$ - ^.gitreview$ - ^.mailmap$ + - tempest-extra-tests: + irrelevant-files: *tempest-irrelevant-files - tempest-full-ubuntu-focal: irrelevant-files: *tempest-irrelevant-files - glance-multistore-cinder-import: @@ -63,6 +65,7 @@ - ^tools/tempest-integrated-gate-placement-exclude-list.txt - ^tools/tempest-integrated-gate-storage-blacklist.txt - ^tools/tempest-integrated-gate-storage-exclude-list.txt + - ^tools/tempest-extra-tests-list.txt - ^tools/verify-ipv6-only-deployments.sh - ^tools/with_venv.sh # tools/ is not here since this relies on a script in tools/. @@ -86,6 +89,7 @@ - ^tools/tempest-integrated-gate-placement-exclude-list.txt - ^tools/tempest-integrated-gate-storage-blacklist.txt - ^tools/tempest-integrated-gate-storage-exclude-list.txt + - ^tools/tempest-extra-tests-list.txt - ^tools/tempest-plugin-sanity.sh - ^tools/with_venv.sh - ^.coveragerc$ @@ -131,6 +135,8 @@ irrelevant-files: *tempest-irrelevant-files - tempest-full-py3: irrelevant-files: *tempest-irrelevant-files + - tempest-extra-tests: + irrelevant-files: *tempest-irrelevant-files - grenade: irrelevant-files: *tempest-irrelevant-files - tempest-ipv6-only: @@ -177,6 +183,9 @@ - tempest-slow-zed - tempest-slow-yoga - tempest-slow-xena + - tempest-full-zed-extra-tests + - tempest-full-yoga-extra-tests + - tempest-full-xena-extra-tests periodic: jobs: - tempest-all diff --git a/zuul.d/stable-jobs.yaml b/zuul.d/stable-jobs.yaml index fb2300b252..f70e79cce5 100644 --- a/zuul.d/stable-jobs.yaml +++ b/zuul.d/stable-jobs.yaml @@ -17,6 +17,24 @@ nodeset: openstack-single-node-focal override-checkout: stable/xena +- job: + name: tempest-full-zed-extra-tests + parent: tempest-extra-tests + nodeset: openstack-single-node-focal + override-checkout: stable/zed + +- job: + name: tempest-full-yoga-extra-tests + parent: tempest-extra-tests + nodeset: openstack-single-node-focal + override-checkout: stable/yoga + +- job: + name: tempest-full-xena-extra-tests + parent: tempest-extra-tests + nodeset: openstack-single-node-focal + override-checkout: stable/xena + - job: name: tempest-slow-zed parent: tempest-slow-py3 diff --git a/zuul.d/tempest-specific.yaml b/zuul.d/tempest-specific.yaml index 684270a5d9..972123ee95 100644 --- a/zuul.d/tempest-specific.yaml +++ b/zuul.d/tempest-specific.yaml @@ -30,6 +30,8 @@ - opendev.org/openstack/oslo.utils - opendev.org/openstack/oslo.versionedobjects - opendev.org/openstack/oslo.vmware + vars: + tox_envlist: full - job: name: tempest-full-parallel