Separate the extra tests to run in a separate job

Recently we are seeing a lot of job timeout(bug#2004780)
and we see many tests taking time and also number of tests
increasing over time. This is to prepare the list of extra tests
(here extra tests means the tests which are covered by the other
API or scenario tests) which we do not need to run in every
integrated jobs. Instead, we can run them in a separete job(s).

Currently I am adding admin (except keystone) and negative tests
in the 'extra tests' list but we can add more tests here which
we think are covered in some other tests.

As negative tests are important for interop, adding those extra
tests coverage for stable branch job also but running them in
periodic run only.

Related-Bug: #2004780
Change-Id: Id02221df0d6180519751c63e890851bd59fdafa0

tools/tempest-extra-tests-list.txt

@@ -0,0 +1,20 @@
+# This file includes the list of tests which need to be
+# excluded to run from integrated testing (tempest-full job
+# or other generic jobs. We will run these tests in a separate
+# jobs. This is needed to avoid the job timeout, details in
+# bug#2004780.
+# Basic criteria to add test in this list is:
+# * Admin test which are not needed for interop and most of them
+#   are running as part of other API and Scenario tests.
+# * Negative tests which are mostly covered in tempest API tests
+#   or service unit/functional tests.
+
+# All admin tests except keystone admin test which might not have much
+# coverage in existing other tests
+tempest.api.compute.admin
+tempest.api.volume.admin
+tempest.api.image.admin
+tempest.api.network.admin
+
+# All negative tests
+negative

tox.ini

@@ -126,17 +126,49 @@ commands =
     tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' {posargs}
     tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)' {posargs}
 
+[testenv:integrated-full]
+envdir = .tox/tempest
+sitepackages = {[tempestenv]sitepackages}
+basepython = {[tempestenv]basepython}
+setenv = {[tempestenv]setenv}
+deps = {[tempestenv]deps}
+# The regex below is used to select which tests to run. It exclude the extra
+# tests mentioned in tools/tempest-extra-tests-list.txt and slow tag:
+# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610
+# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now.
+regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
+regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)'
+commands =
+    find . -type f -name "*.pyc" -delete
+    tempest run --regex {[testenv:integrated-full]regex1} --exclude-list ./tools/tempest-extra-tests-list.txt {posargs}
+    tempest run --combine --serial --regex {[testenv:integrated-full]regex2} {posargs}
+
+[testenv:extra-tests]
+envdir = .tox/tempest
+sitepackages = {[tempestenv]sitepackages}
+basepython = {[tempestenv]basepython}
+setenv = {[tempestenv]setenv}
+deps = {[tempestenv]deps}
+# The regex below is used to select extra tests mentioned in
+# tools/tempest-extra-tests-list.txt and exclude slow tag tests:
+# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610
+# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now.
+exclude-regex = '\[.*\bslow\b.*\]'
+commands =
+    find . -type f -name "*.pyc" -delete
+    tempest run --exclude-regex {[testenv:extra-tests]exclude-regex} --include-list ./tools/tempest-extra-tests-list.txt {posargs}
+
 [testenv:full-parallel]
 envdir = .tox/tempest
 sitepackages = {[tempestenv]sitepackages}
 basepython = {[tempestenv]basepython}
 setenv = {[tempestenv]setenv}
 deps = {[tempestenv]deps}
+# But exlcude the extra tests mentioned in tools/tempest-extra-tests-list.txt
 regex = '(^tempest\.scenario.*)|(^tempest\.serial_tests)|(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
-# The regex below is used to select all tempest scenario and including the non slow api tests
 commands =
     find . -type f -name "*.pyc" -delete
-    tempest run --regex {[testenv:full-parallel]regex} {posargs}
+    tempest run --regex {[testenv:full-parallel]regex} --exclude-list ./tools/tempest-extra-tests-list.txt {posargs}
 
 [testenv:api-microversion-tests]
 envdir = .tox/tempest

zuul.d/integrated-gate.yaml

@@ -59,6 +59,15 @@
         # tests fail at a high rate (see bugs 1483434, 1813217, 1745168)
         c-bak: false
 
+- job:
+    name: tempest-extra-tests
+    parent: devstack-tempest
+    description: |
+      This job runs the extra tests mentioned in
+      tools/tempest-extra-tests-list.txt.
+    vars:
+      tox_envlist: extra-tests
+
 - job:
     name: tempest-full-py3
     parent: devstack-tempest
@@ -74,7 +83,7 @@
     required-projects:
       - openstack/horizon
     vars:
-      tox_envlist: full
+      tox_envlist: integrated-full
       devstack_localrc:
         USE_PYTHON3: true
         FORCE_CONFIG_DRIVE: true
@@ -107,6 +116,7 @@
       # Required until bug/1949606 is resolved when using libvirt and QEMU
       # >=5.0.0 with a [libvirt]virt_type of qemu (TCG).
       configure_swap_size: 4096
+      tox_envlist: full
 
 - job:
     name: tempest-integrated-networking

zuul.d/project.yaml

@@ -26,6 +26,8 @@
               - ^.gitignore$
               - ^.gitreview$
               - ^.mailmap$
+        - tempest-extra-tests:
+            irrelevant-files: *tempest-irrelevant-files
         - tempest-full-ubuntu-focal:
             irrelevant-files: *tempest-irrelevant-files
         - glance-multistore-cinder-import:
@@ -63,6 +65,7 @@
               - ^tools/tempest-integrated-gate-placement-exclude-list.txt
               - ^tools/tempest-integrated-gate-storage-blacklist.txt
               - ^tools/tempest-integrated-gate-storage-exclude-list.txt
+              - ^tools/tempest-extra-tests-list.txt
               - ^tools/verify-ipv6-only-deployments.sh
               - ^tools/with_venv.sh
               # tools/ is not here since this relies on a script in tools/.
@@ -86,6 +89,7 @@
               - ^tools/tempest-integrated-gate-placement-exclude-list.txt
               - ^tools/tempest-integrated-gate-storage-blacklist.txt
               - ^tools/tempest-integrated-gate-storage-exclude-list.txt
+              - ^tools/tempest-extra-tests-list.txt
               - ^tools/tempest-plugin-sanity.sh
               - ^tools/with_venv.sh
               - ^.coveragerc$
@@ -131,6 +135,8 @@
             irrelevant-files: *tempest-irrelevant-files
         - tempest-full-py3:
             irrelevant-files: *tempest-irrelevant-files
+        - tempest-extra-tests:
+            irrelevant-files: *tempest-irrelevant-files
         - grenade:
             irrelevant-files: *tempest-irrelevant-files
         - tempest-ipv6-only:
@@ -177,6 +183,9 @@
         - tempest-slow-zed
         - tempest-slow-yoga
         - tempest-slow-xena
+        - tempest-full-zed-extra-tests
+        - tempest-full-yoga-extra-tests
+        - tempest-full-xena-extra-tests
     periodic:
       jobs:
         - tempest-all

zuul.d/stable-jobs.yaml

@@ -17,6 +17,24 @@
     nodeset: openstack-single-node-focal
     override-checkout: stable/xena
 
+- job:
+    name: tempest-full-zed-extra-tests
+    parent: tempest-extra-tests
+    nodeset: openstack-single-node-focal
+    override-checkout: stable/zed
+
+- job:
+    name: tempest-full-yoga-extra-tests
+    parent: tempest-extra-tests
+    nodeset: openstack-single-node-focal
+    override-checkout: stable/yoga
+
+- job:
+    name: tempest-full-xena-extra-tests
+    parent: tempest-extra-tests
+    nodeset: openstack-single-node-focal
+    override-checkout: stable/xena
+
 - job:
     name: tempest-slow-zed
     parent: tempest-slow-py3

zuul.d/tempest-specific.yaml

@@ -30,6 +30,8 @@
       - opendev.org/openstack/oslo.utils
       - opendev.org/openstack/oslo.versionedobjects
       - opendev.org/openstack/oslo.vmware
+    vars:
+      tox_envlist: full
 
 - job:
     name: tempest-full-parallel