Merge "Separate the extra tests to run in a separate job"
This commit is contained in:
Zuul
@@ -0,0 +1,20 @@
|
||||
# This file includes the list of tests which need to be
|
||||
# excluded to run from integrated testing (tempest-full job
|
||||
# or other generic jobs. We will run these tests in a separate
|
||||
# jobs. This is needed to avoid the job timeout, details in
|
||||
# bug#2004780.
|
||||
# Basic criteria to add test in this list is:
|
||||
# * Admin test which are not needed for interop and most of them
|
||||
# are running as part of other API and Scenario tests.
|
||||
# * Negative tests which are mostly covered in tempest API tests
|
||||
# or service unit/functional tests.
|
||||
|
||||
# All admin tests except keystone admin test which might not have much
|
||||
# coverage in existing other tests
|
||||
tempest.api.compute.admin
|
||||
tempest.api.volume.admin
|
||||
tempest.api.image.admin
|
||||
tempest.api.network.admin
|
||||
|
||||
# All negative tests
|
||||
negative
|
||||
@@ -126,17 +126,49 @@ commands =
|
||||
tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' {posargs}
|
||||
tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)' {posargs}
|
||||
|
||||
[testenv:integrated-full]
|
||||
envdir = .tox/tempest
|
||||
sitepackages = {[tempestenv]sitepackages}
|
||||
basepython = {[tempestenv]basepython}
|
||||
setenv = {[tempestenv]setenv}
|
||||
deps = {[tempestenv]deps}
|
||||
# The regex below is used to select which tests to run. It exclude the extra
|
||||
# tests mentioned in tools/tempest-extra-tests-list.txt and slow tag:
|
||||
# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610
|
||||
# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now.
|
||||
regex1 = '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
|
||||
regex2 = '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)|(^tempest\.serial_tests)'
|
||||
commands =
|
||||
find . -type f -name "*.pyc" -delete
|
||||
tempest run --regex {[testenv:integrated-full]regex1} --exclude-list ./tools/tempest-extra-tests-list.txt {posargs}
|
||||
tempest run --combine --serial --regex {[testenv:integrated-full]regex2} {posargs}
|
||||
|
||||
[testenv:extra-tests]
|
||||
envdir = .tox/tempest
|
||||
sitepackages = {[tempestenv]sitepackages}
|
||||
basepython = {[tempestenv]basepython}
|
||||
setenv = {[tempestenv]setenv}
|
||||
deps = {[tempestenv]deps}
|
||||
# The regex below is used to select extra tests mentioned in
|
||||
# tools/tempest-extra-tests-list.txt and exclude slow tag tests:
|
||||
# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610
|
||||
# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now.
|
||||
exclude-regex = '\[.*\bslow\b.*\]'
|
||||
commands =
|
||||
find . -type f -name "*.pyc" -delete
|
||||
tempest run --exclude-regex {[testenv:extra-tests]exclude-regex} --include-list ./tools/tempest-extra-tests-list.txt {posargs}
|
||||
|
||||
[testenv:full-parallel]
|
||||
envdir = .tox/tempest
|
||||
sitepackages = {[tempestenv]sitepackages}
|
||||
basepython = {[tempestenv]basepython}
|
||||
setenv = {[tempestenv]setenv}
|
||||
deps = {[tempestenv]deps}
|
||||
# But exlcude the extra tests mentioned in tools/tempest-extra-tests-list.txt
|
||||
regex = '(^tempest\.scenario.*)|(^tempest\.serial_tests)|(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
|
||||
# The regex below is used to select all tempest scenario and including the non slow api tests
|
||||
commands =
|
||||
find . -type f -name "*.pyc" -delete
|
||||
tempest run --regex {[testenv:full-parallel]regex} {posargs}
|
||||
tempest run --regex {[testenv:full-parallel]regex} --exclude-list ./tools/tempest-extra-tests-list.txt {posargs}
|
||||
|
||||
[testenv:api-microversion-tests]
|
||||
envdir = .tox/tempest
|
||||
|
||||
@@ -59,6 +59,15 @@
|
||||
# tests fail at a high rate (see bugs 1483434, 1813217, 1745168)
|
||||
c-bak: false
|
||||
|
||||
- job:
|
||||
name: tempest-extra-tests
|
||||
parent: devstack-tempest
|
||||
description: |
|
||||
This job runs the extra tests mentioned in
|
||||
tools/tempest-extra-tests-list.txt.
|
||||
vars:
|
||||
tox_envlist: extra-tests
|
||||
|
||||
- job:
|
||||
name: tempest-full-py3
|
||||
parent: devstack-tempest
|
||||
@@ -74,7 +83,7 @@
|
||||
required-projects:
|
||||
- openstack/horizon
|
||||
vars:
|
||||
tox_envlist: full
|
||||
tox_envlist: integrated-full
|
||||
devstack_localrc:
|
||||
USE_PYTHON3: true
|
||||
FORCE_CONFIG_DRIVE: true
|
||||
@@ -107,6 +116,7 @@
|
||||
# Required until bug/1949606 is resolved when using libvirt and QEMU
|
||||
# >=5.0.0 with a [libvirt]virt_type of qemu (TCG).
|
||||
configure_swap_size: 4096
|
||||
tox_envlist: full
|
||||
|
||||
- job:
|
||||
name: tempest-integrated-networking
|
||||
|
||||
@@ -26,6 +26,8 @@
|
||||
- ^.gitignore$
|
||||
- ^.gitreview$
|
||||
- ^.mailmap$
|
||||
- tempest-extra-tests:
|
||||
irrelevant-files: *tempest-irrelevant-files
|
||||
- tempest-full-ubuntu-focal:
|
||||
irrelevant-files: *tempest-irrelevant-files
|
||||
- glance-multistore-cinder-import:
|
||||
@@ -63,6 +65,7 @@
|
||||
- ^tools/tempest-integrated-gate-placement-exclude-list.txt
|
||||
- ^tools/tempest-integrated-gate-storage-blacklist.txt
|
||||
- ^tools/tempest-integrated-gate-storage-exclude-list.txt
|
||||
- ^tools/tempest-extra-tests-list.txt
|
||||
- ^tools/verify-ipv6-only-deployments.sh
|
||||
- ^tools/with_venv.sh
|
||||
# tools/ is not here since this relies on a script in tools/.
|
||||
@@ -86,6 +89,7 @@
|
||||
- ^tools/tempest-integrated-gate-placement-exclude-list.txt
|
||||
- ^tools/tempest-integrated-gate-storage-blacklist.txt
|
||||
- ^tools/tempest-integrated-gate-storage-exclude-list.txt
|
||||
- ^tools/tempest-extra-tests-list.txt
|
||||
- ^tools/tempest-plugin-sanity.sh
|
||||
- ^tools/with_venv.sh
|
||||
- ^.coveragerc$
|
||||
@@ -131,6 +135,8 @@
|
||||
irrelevant-files: *tempest-irrelevant-files
|
||||
- tempest-full-py3:
|
||||
irrelevant-files: *tempest-irrelevant-files
|
||||
- tempest-extra-tests:
|
||||
irrelevant-files: *tempest-irrelevant-files
|
||||
- grenade:
|
||||
irrelevant-files: *tempest-irrelevant-files
|
||||
- tempest-ipv6-only:
|
||||
@@ -179,6 +185,9 @@
|
||||
- tempest-slow-zed
|
||||
- tempest-slow-yoga
|
||||
- tempest-slow-xena
|
||||
- tempest-full-zed-extra-tests
|
||||
- tempest-full-yoga-extra-tests
|
||||
- tempest-full-xena-extra-tests
|
||||
periodic:
|
||||
jobs:
|
||||
- tempest-all
|
||||
|
||||
@@ -17,6 +17,24 @@
|
||||
nodeset: openstack-single-node-focal
|
||||
override-checkout: stable/xena
|
||||
|
||||
- job:
|
||||
name: tempest-full-zed-extra-tests
|
||||
parent: tempest-extra-tests
|
||||
nodeset: openstack-single-node-focal
|
||||
override-checkout: stable/zed
|
||||
|
||||
- job:
|
||||
name: tempest-full-yoga-extra-tests
|
||||
parent: tempest-extra-tests
|
||||
nodeset: openstack-single-node-focal
|
||||
override-checkout: stable/yoga
|
||||
|
||||
- job:
|
||||
name: tempest-full-xena-extra-tests
|
||||
parent: tempest-extra-tests
|
||||
nodeset: openstack-single-node-focal
|
||||
override-checkout: stable/xena
|
||||
|
||||
- job:
|
||||
name: tempest-slow-zed
|
||||
parent: tempest-slow-py3
|
||||
|
||||
@@ -30,6 +30,8 @@
|
||||
- opendev.org/openstack/oslo.utils
|
||||
- opendev.org/openstack/oslo.versionedobjects
|
||||
- opendev.org/openstack/oslo.vmware
|
||||
vars:
|
||||
tox_envlist: full
|
||||
|
||||
- job:
|
||||
name: tempest-full-parallel
|
||||
|
||||
Reference in New Issue
Block a user