openstack
/
tempest
Code Issues Proposed changes

Merge "Inclusive jargon"

changes/64/774764/1
Zuul 2021-02-09 22:33:47 +00:00 committed by Gerrit Code Review
parent 739a9f8222 dc84423b72
commit eae384ac1d
21 changed files with 350 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/source/data/tempest-blacklisted-plugins-registry.header → doc/source/data/tempest-non-active-plugins-registry.header
View File

@ -1,7 +1,7 @@
Blacklisted Plugins
Non Active Plugins
===================
List of Tempest plugin projects that are stale or unmaintained for a long
time (6 months or more). They can be moved out of blacklist state once one
time (6 months or more). They can be moved out of nonactivelist state once one
of the relevant patches gets merged:
https://review.opendev.org/#/q/topic:tempest-sanity-gate+%28status:open%29

2
doc/source/overview.rst
View File

@ -113,7 +113,7 @@ as it is simpler, and quicker to work with.
There is also the option to use `stestr`_ directly. For example, from
the workspace dir run::
$ stestr run --black-regex '\[.*\bslow\b.*\]' '^tempest\.(api|scenario)'
$ stestr run --exclude-regex '\[.*\bslow\b.*\]' '^tempest\.(api|scenario)'
will run the same set of tests as the default gate jobs. Or you can
use `unittest`_ compatible test runners such as `stestr`_, `pytest`_ etc.

2
doc/source/stable_branch_support_policy.rst
View File

@ -20,7 +20,7 @@ Tempest from master as long as possible. But, because we won't be actively
testing branches in these phases, it's possible that we'll introduce changes to
Tempest on master which will break support on *Extended Maintenance* phase
branches. When this happens the expectation for those branches is to either
switch to running Tempest from a tag with support for the branch, or blacklist
switch to running Tempest from a tag with support for the branch, or exclude
a newly introduced test (if that is the cause of the issue). Tempest will not
be creating stable branches to support *Extended Maintenance* phase branches, as
the burden is on the *Extended Maintenance* phase branche maintainers, not the Tempest

0
etc/whitelist.yaml → etc/allow-list.yaml
View File

13
releasenotes/notes/Inclusive-jargon-17621346744f0cf4.yaml Normal file
View File

@ -0,0 +1,13 @@
---
deprecations:
- |
In this release the following tempest arguments are deprecated and
replaced by new ones which are functionally equivalent:
* --black-regex is replaced by --exclude-regex
* --blacklist-file is replaced by --exclude-list
* --whitelist-file is replaced by --include-list
For now Tempest supports both (new and old ones) in order to make the
transition for all consumers smoother. However, that's just a temporary
case and the old options will be removed soon.

13
roles/run-tempest/README.rst
View File

@ -32,7 +32,11 @@ variable (through the `register` statement).
.. zuul:rolevar:: tempest_test_blacklist
Specifies a blacklist file to skip tests that are not needed.
DEPRECATED option, please use tempest_test_exclude_list instead.
.. zuul:rolevar:: tempest_test_exclude_list
Specifies an excludelist file to skip tests that are not needed.
Pass a full path to the file.
@ -44,6 +48,11 @@ variable (through the `register` statement).
.. zuul:rolevar:: tempest_black_regex
:default: ''
DEPRECATED option, please use tempest_exclude_regex instead.
.. zuul:rolevar:: tempest_exclude_regex
:default: ''
A regular expression used to skip the tests.
It works only when used with some specific tox environments
@ -51,7 +60,7 @@ variable (through the `register` statement).
::
vars:
tempest_black_regex: (tempest.api.identity).*$
tempest_exclude_regex: (tempest.api.identity).*$
.. zuul:rolevar:: tox_extra_args
:default: ''

3
roles/run-tempest/defaults/main.yaml
View File

@ -1,7 +1,10 @@
devstack_base_dir: /opt/stack
tempest_test_regex: ''
tox_envlist: smoke
# TODO(kopecmartin) remove tempest_black_regex once all consumers of this
# role have switched to the tempest_exclude_regex option.
tempest_black_regex: ''
tempest_exclude_regex: ''
tox_extra_args: ''
tempest_test_timeout: ''
stable_constraints_file: "{{ devstack_base_dir }}/requirements/upper-constraints.txt"

45
roles/run-tempest/tasks/main.yaml
View File

@ -36,6 +36,9 @@
tempest_tox_environment: "{{ tempest_tox_environment | combine({'OS_TEST_TIMEOUT': tempest_test_timeout}) }}"
when: tempest_test_timeout != ''
# TODO(kopecmartin) remove the following 'when block' after all consumers of
# the role have switched to tempest_test_exclude_list option, until then it's
# kept here for backward compatibility
- when:
- tempest_test_blacklist is defined
block:
@ -50,10 +53,48 @@
blacklist_option: "--blacklist-file={{ tempest_test_blacklist|quote }}"
when: blacklist_stat.stat.exists
- when:
- tempest_test_exclude_list is defined
block:
- name: Check for test exclude list file
stat:
path: "{{ tempest_test_exclude_list }}"
register:
exclude_list_stat
- name: Build exclude list option
set_fact:
exclude_list_option: "--exclude-list={{ tempest_test_exclude_list|quote }}"
when: exclude_list_stat.stat.exists
# TODO(kopecmartin) remove this after all consumers of the role have switched
# to tempest_exclude_regex option, until then it's kept here for the backward
# compatibility
- name: Set tempest_exclude_regex
set_fact:
tempest_exclude_regex: "{{ tempest_black_regex }}"
when:
- tempest_black_regex is defined
- tempest_exclude_regex is not defined
- name: Build exclude regex (old param)
set_fact:
tempest_exclude_regex: "--black-regex={{tempest_black_regex|quote}}"
when:
- tempest_black_regex is defined
- name: Build exclude regex (new param)
set_fact:
tempest_exclude_regex: "--exclude-regex={{tempest_exclude_regex|quote}}"
when:
- tempest_black_regex is not defined
- tempest_exclude_regex is defined
- name: Run Tempest
command: tox -e {{tox_envlist}} {{tox_extra_args}} -- {{tempest_test_regex|quote}} {{blacklist_option|default('')}} \
command: tox -e {{tox_envlist}} {{tox_extra_args}} -- {{tempest_test_regex|quote}} \
{{blacklist_option|default('')}} {{exclude_list_option|default('')}} \
--concurrency={{tempest_concurrency|default(default_concurrency)}} \
--black-regex={{tempest_black_regex|quote}}
{{tempest_exclude_regex|default('')}}
args:
chdir: "{{devstack_base_dir}}/tempest"
register: tempest_run_result

116
tempest/cmd/run.py
View File

@ -22,10 +22,10 @@ Tempest run has several options:
* ``--regex/-r``: This is a selection regex like what stestr uses. It will run
any tests that match on re.match() with the regex
* ``--smoke/-s``: Run all the tests tagged as smoke
* ``--black-regex``: It allows to do simple test exclusion via passing a
rejection/black regexp
* ``--exclude-regex``: It allows to do simple test exclusion via passing a
rejection/exclude regexp
There are also the ``--blacklist-file`` and ``--whitelist-file`` options that
There are also the ``--exclude-list`` and ``--include-list`` options that
let you pass a filepath to tempest run with the file format being a line
separated regex, with '#' used to signify the start of a comment on a line.
For example::
@ -128,6 +128,7 @@ import os
import sys
from cliff import command
from oslo_log import log
from oslo_serialization import jsonutils as json
from stestr import commands
@ -141,6 +142,8 @@ from tempest import config
CONF = config.CONF
SAVED_STATE_JSON = "saved_state.json"
LOG = log.getLogger(__name__)
class TempestRun(command.Command):
@ -201,23 +204,71 @@ class TempestRun(command.Command):
self._init_state()
regex = self._build_regex(parsed_args)
# temporary method for parsing deprecated and new stestr options
# and showing warning messages in order to make the transition
# smoother for all tempest consumers
# TODO(kopecmartin) remove this after stestr>=3.1.0 is used
# in all supported OpenStack releases
def parse_dep(old_o, old_v, new_o, new_v):
ret = ''
if old_v:
LOG.warning("'%s' option is deprecated, use '%s' instead "
"which is functionally equivalent. Right now "
"Tempest still supports this option for "
"backward compatibility, however, it will be "
"removed soon.",
old_o, new_o)
ret = old_v
if old_v and new_v:
# both options are specified
LOG.warning("'%s' and '%s' are specified at the same time, "
"'%s' takes precedence over '%s'",
new_o, old_o, new_o, old_o)
if new_v:
ret = new_v
return ret
ex_regex = parse_dep('--black-regex', parsed_args.black_regex,
'--exclude-regex', parsed_args.exclude_regex)
ex_list = parse_dep('--blacklist-file', parsed_args.blacklist_file,
'--exclude-list', parsed_args.exclude_list)
in_list = parse_dep('--whitelist-file', parsed_args.whitelist_file,
'--include-list', parsed_args.include_list)
return_code = 0
if parsed_args.list_tests:
return_code = commands.list_command(
filters=regex, whitelist_file=parsed_args.whitelist_file,
blacklist_file=parsed_args.blacklist_file,
black_regex=parsed_args.black_regex)
try:
return_code = commands.list_command(
filters=regex, include_list=in_list,
exclude_list=ex_list, exclude_regex=ex_regex)
except TypeError:
# exclude_list, include_list and exclude_regex are defined only
# in stestr >= 3.1.0, this except block catches the case when
# tempest is executed with an older stestr
return_code = commands.list_command(
filters=regex, whitelist_file=in_list,
blacklist_file=ex_list, black_regex=ex_regex)
else:
serial = not parsed_args.parallel
return_code = commands.run_command(
filters=regex, subunit_out=parsed_args.subunit,
serial=serial, concurrency=parsed_args.concurrency,
blacklist_file=parsed_args.blacklist_file,
whitelist_file=parsed_args.whitelist_file,
black_regex=parsed_args.black_regex,
worker_path=parsed_args.worker_file,
load_list=parsed_args.load_list, combine=parsed_args.combine)
params = {
'filters': regex, 'subunit_out': parsed_args.subunit,
'serial': serial, 'concurrency': parsed_args.concurrency,
'worker_path': parsed_args.worker_file,
'load_list': parsed_args.load_list,
'combine': parsed_args.combine
}
try:
return_code = commands.run_command(
**params, exclude_list=ex_list,
include_list=in_list, exclude_regex=ex_regex)
except TypeError:
# exclude_list, include_list and exclude_regex are defined only
# in stestr >= 3.1.0, this except block catches the case when
# tempest is executed with an older stestr
return_code = commands.run_command(
**params, blacklist_file=ex_list,
whitelist_file=in_list, black_regex=ex_regex)
if return_code > 0:
sys.exit(return_code)
return return_code
@ -271,15 +322,38 @@ class TempestRun(command.Command):
help='A normal stestr selection regex used to '
'specify a subset of tests to run')
parser.add_argument('--black-regex', dest='black_regex',
help='DEPRECATED: This option is deprecated and '
'will be removed soon, use --exclude-regex '
'which is functionally equivalent. If this '
'is specified at the same time as '
'--exclude-regex, this flag will be ignored '
'and --exclude-regex will be used')
parser.add_argument('--exclude-regex', dest='exclude_regex',
help='A regex to exclude tests that match it')
parser.add_argument('--whitelist-file', '--whitelist_file',
help="Path to a whitelist file, this file "
"contains a separate regex on each "
"newline.")
help='DEPRECATED: This option is deprecated and '
'will be removed soon, use --include-list '
'which is functionally equivalent. If this '
'is specified at the same time as '
'--include-list, this flag will be ignored '
'and --include-list will be used')
parser.add_argument('--include-list', '--include_list',
help="Path to an include file which contains the "
"regex for tests to be included in tempest "
"run, this file contains a separate regex on "
"each newline.")
parser.add_argument('--blacklist-file', '--blacklist_file',
help='Path to a blacklist file, this file '
'contains a separate regex exclude on '
'each newline')
help='DEPRECATED: This option is deprecated and '
'will be removed soon, use --exclude-list '
'which is functionally equivalent. If this '
'is specified at the same time as '
'--exclude-list, this flag will be ignored '
'and --exclude-list will be used')
parser.add_argument('--exclude-list', '--exclude_list',
help='Path to an exclude file which contains the '
'regex for tests to be excluded in tempest '
'run, this file contains a separate regex on '
'each newline.')
parser.add_argument('--load-list', '--load_list',
help='Path to a non-regex whitelist file, '
'this file contains a separate test '

180
tempest/tests/cmd/test_run.py
View File

@ -68,6 +68,11 @@ class TestTempestRun(base.TestCase):
class TestRunReturnCode(base.TestCase):
exclude_regex = '--exclude-regex'
exclude_list = '--exclude-list'
include_list = '--include-list'
def setUp(self):
super(TestRunReturnCode, self).setUp()
# Setup test dirs
@ -92,6 +97,14 @@ class TestRunReturnCode(base.TestCase):
self.addCleanup(os.chdir, os.path.abspath(os.curdir))
os.chdir(self.directory)
def _get_test_list_file(self, content):
fd, path = tempfile.mkstemp()
self.addCleanup(os.remove, path)
test_file = os.fdopen(fd, 'wb', 0)
self.addCleanup(test_file.close)
test_file.write(content.encode('utf-8'))
return path
def assertRunExit(self, cmd, expected):
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
@ -115,19 +128,23 @@ class TestRunReturnCode(base.TestCase):
subprocess.call(['stestr', 'init'])
self.assertRunExit(['tempest', 'run', '--regex', 'failing'], 1)
def test_tempest_run_blackregex_failing(self):
self.assertRunExit(['tempest', 'run', '--black-regex', 'failing'], 0)
def test_tempest_run_exclude_regex_failing(self):
self.assertRunExit(['tempest', 'run',
self.exclude_regex, 'failing'], 0)
def test_tempest_run_blackregex_failing_with_stestr_repository(self):
def test_tempest_run_exclude_regex_failing_with_stestr_repository(self):
subprocess.call(['stestr', 'init'])
self.assertRunExit(['tempest', 'run', '--black-regex', 'failing'], 0)
self.assertRunExit(['tempest', 'run',
self.exclude_regex, 'failing'], 0)
def test_tempest_run_blackregex_passing(self):
self.assertRunExit(['tempest', 'run', '--black-regex', 'passing'], 1)
def test_tempest_run_exclude_regex_passing(self):
self.assertRunExit(['tempest', 'run',
self.exclude_regex, 'passing'], 1)
def test_tempest_run_blackregex_passing_with_stestr_repository(self):
def test_tempest_run_exclude_regex_passing_with_stestr_repository(self):
subprocess.call(['stestr', 'init'])
self.assertRunExit(['tempest', 'run', '--black-regex', 'passing'], 1)
self.assertRunExit(['tempest', 'run',
self.exclude_regex, 'passing'], 1)
def test_tempest_run_fails(self):
self.assertRunExit(['tempest', 'run'], 1)
@ -149,47 +166,31 @@ class TestRunReturnCode(base.TestCase):
self.assertEqual(result, tests)
def test_tempest_run_with_worker_file(self):
fd, path = tempfile.mkstemp()
self.addCleanup(os.remove, path)
worker_file = os.fdopen(fd, 'wb', 0)
self.addCleanup(worker_file.close)
worker_file.write(
'- worker:\n - passing\n concurrency: 3'.encode('utf-8'))
path = self._get_test_list_file(
'- worker:\n - passing\n concurrency: 3')
self.assertRunExit(['tempest', 'run', '--worker-file=%s' % path], 0)
def test_tempest_run_with_whitelist(self):
fd, path = tempfile.mkstemp()
self.addCleanup(os.remove, path)
whitelist_file = os.fdopen(fd, 'wb', 0)
self.addCleanup(whitelist_file.close)
whitelist_file.write('passing'.encode('utf-8'))
self.assertRunExit(['tempest', 'run', '--whitelist-file=%s' % path], 0)
def test_tempest_run_with_include_list(self):
path = self._get_test_list_file('passing')
self.assertRunExit(['tempest', 'run',
'%s=%s' % (self.include_list, path)], 0)
def test_tempest_run_with_whitelist_regex_include_pass_check_fail(self):
fd, path = tempfile.mkstemp()
self.addCleanup(os.remove, path)
whitelist_file = os.fdopen(fd, 'wb', 0)
self.addCleanup(whitelist_file.close)
whitelist_file.write('passing'.encode('utf-8'))
self.assertRunExit(['tempest', 'run', '--whitelist-file=%s' % path,
def test_tempest_run_with_include_regex_include_pass_check_fail(self):
path = self._get_test_list_file('passing')
self.assertRunExit(['tempest', 'run',
'%s=%s' % (self.include_list, path),
'--regex', 'fail'], 1)
def test_tempest_run_with_whitelist_regex_include_pass_check_pass(self):
fd, path = tempfile.mkstemp()
self.addCleanup(os.remove, path)
whitelist_file = os.fdopen(fd, 'wb', 0)
self.addCleanup(whitelist_file.close)
whitelist_file.write('passing'.encode('utf-8'))
self.assertRunExit(['tempest', 'run', '--whitelist-file=%s' % path,
def test_tempest_run_with_include_regex_include_pass_check_pass(self):
path = self._get_test_list_file('passing')
self.assertRunExit(['tempest', 'run',
'%s=%s' % (self.include_list, path),
'--regex', 'passing'], 0)
def test_tempest_run_with_whitelist_regex_include_fail_check_pass(self):
fd, path = tempfile.mkstemp()
self.addCleanup(os.remove, path)
whitelist_file = os.fdopen(fd, 'wb', 0)
self.addCleanup(whitelist_file.close)
whitelist_file.write('failing'.encode('utf-8'))
self.assertRunExit(['tempest', 'run', '--whitelist-file=%s' % path,
def test_tempest_run_with_include_regex_include_fail_check_pass(self):
path = self._get_test_list_file('failing')
self.assertRunExit(['tempest', 'run',
'%s=%s' % (self.include_list, path),
'--regex', 'pass'], 1)
def test_tempest_run_passes_with_config_file(self):
@ -197,50 +198,75 @@ class TestRunReturnCode(base.TestCase):
'--config-file', self.stestr_conf_file,
'--regex', 'passing'], 0)
def test_tempest_run_with_blacklist_failing(self):
fd, path = tempfile.mkstemp()
self.addCleanup(os.remove, path)
blacklist_file = os.fdopen(fd, 'wb', 0)
self.addCleanup(blacklist_file.close)
blacklist_file.write('failing'.encode('utf-8'))
self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path], 0)
def test_tempest_run_with_exclude_list_failing(self):
path = self._get_test_list_file('failing')
self.assertRunExit(['tempest', 'run',
'%s=%s' % (self.exclude_list, path)], 0)
def test_tempest_run_with_blacklist_passing(self):
fd, path = tempfile.mkstemp()
self.addCleanup(os.remove, path)
blacklist_file = os.fdopen(fd, 'wb', 0)
self.addCleanup(blacklist_file.close)
blacklist_file.write('passing'.encode('utf-8'))
self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path], 1)
def test_tempest_run_with_exclude_list_passing(self):
path = self._get_test_list_file('passing')
self.assertRunExit(['tempest', 'run',
'%s=%s' % (self.exclude_list, path)], 1)
def test_tempest_run_with_blacklist_regex_exclude_fail_check_pass(self):
fd, path = tempfile.mkstemp()
self.addCleanup(os.remove, path)
blacklist_file = os.fdopen(fd, 'wb', 0)
self.addCleanup(blacklist_file.close)
blacklist_file.write('failing'.encode('utf-8'))
self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path,
def test_tempest_run_with_exclude_list_regex_exclude_fail_check_pass(self):
path = self._get_test_list_file('failing')
self.assertRunExit(['tempest', 'run',
'%s=%s' % (self.exclude_list, path),
'--regex', 'pass'], 0)
def test_tempest_run_with_blacklist_regex_exclude_pass_check_pass(self):
fd, path = tempfile.mkstemp()
self.addCleanup(os.remove, path)
blacklist_file = os.fdopen(fd, 'wb', 0)
self.addCleanup(blacklist_file.close)
blacklist_file.write('passing'.encode('utf-8'))
self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path,
def test_tempest_run_with_exclude_list_regex_exclude_pass_check_pass(self):
path = self._get_test_list_file('passing')
self.assertRunExit(['tempest', 'run',
'%s=%s' % (self.exclude_list, path),
'--regex', 'pass'], 1)
def test_tempest_run_with_blacklist_regex_exclude_pass_check_fail(self):
fd, path = tempfile.mkstemp()
self.addCleanup(os.remove, path)
blacklist_file = os.fdopen(fd, 'wb', 0)
self.addCleanup(blacklist_file.close)
blacklist_file.write('passing'.encode('utf-8'))
self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path,
def test_tempest_run_with_exclude_list_regex_exclude_pass_check_fail(self):
path = self._get_test_list_file('passing')
self.assertRunExit(['tempest', 'run',
'%s=%s' % (self.exclude_list, path),
'--regex', 'fail'], 1)
class TestOldArgRunReturnCode(TestRunReturnCode):
"""A class for testing deprecated but still supported args.
This class will be removed once we remove the following arguments:
* --black-regex
* --blacklist-file
* --whitelist-file
"""
exclude_regex = '--black-regex'
exclude_list = '--blacklist-file'
include_list = '--whitelist-file'
def _test_args_passing(self, args):
self.assertRunExit(['tempest', 'run'] + args, 0)
def test_tempest_run_new_old_arg_comb(self):
path = self._get_test_list_file('failing')
self._test_args_passing(['--black-regex', 'failing',
'--exclude-regex', 'failing'])
self._test_args_passing(['--blacklist-file=' + path,
'--exclude-list=' + path])
path = self._get_test_list_file('passing')
self._test_args_passing(['--whitelist-file=' + path,
'--include-list=' + path])
def _test_args_passing_with_stestr_repository(self, args):
subprocess.call(['stestr', 'init'])
self.assertRunExit(['tempest', 'run'] + args, 0)
def test_tempest_run_new_old_arg_comb_with_stestr_repository(self):
path = self._get_test_list_file('failing')
self._test_args_passing_with_stestr_repository(
['--black-regex', 'failing', '--exclude-regex', 'failing'])
self._test_args_passing_with_stestr_repository(
['--blacklist-file=' + path, '--exclude-list=' + path])
path = self._get_test_list_file('passing')
self._test_args_passing_with_stestr_repository(
['--whitelist-file=' + path, '--include-list=' + path])
class TestConfigPathCheck(base.TestCase):
def setUp(self):
super(TestConfigPathCheck, self).setUp()

44
tools/check_logs.py
View File

@ -56,39 +56,39 @@ allowed_dirty = set([
's-proxy'])
def process_files(file_specs, url_specs, whitelists):
def process_files(file_specs, url_specs, allow_lists):
regexp = re.compile(r"^.* (ERROR|CRITICAL|TRACE) .*\[.*\-.*\]")
logs_with_errors = []
for (name, filename) in file_specs:
whitelist = whitelists.get(name, [])
allow_list = allow_lists.get(name, [])
with open(filename) as content:
if scan_content(content, regexp, whitelist):
if scan_content(content, regexp, allow_list):
logs_with_errors.append(name)
for (name, url) in url_specs:
whitelist = whitelists.get(name, [])
allow_list = allow_lists.get(name, [])
req = urlreq.Request(url)
req.add_header('Accept-Encoding', 'gzip')
page = urlreq.urlopen(req)
buf = six.StringIO(page.read())
f = gzip.GzipFile(fileobj=buf)
if scan_content(f.read().splitlines(), regexp, whitelist):
if scan_content(f.read().splitlines(), regexp, allow_list):
logs_with_errors.append(name)
return logs_with_errors
def scan_content(content, regexp, whitelist):
def scan_content(content, regexp, allow_list):
had_errors = False
for line in content:
if not line.startswith("Stderr:") and regexp.match(line):
whitelisted = False
for w in whitelist:
allowed = False
for w in allow_list:
pat = ".*%s.*%s.*" % (w['module'].replace('.', '\\.'),
w['message'])
if re.match(pat, line):
whitelisted = True
allowed = True
break
if not whitelisted or dump_all_errors:
if not whitelisted:
if not allowed or dump_all_errors:
if not allowed:
had_errors = True
return had_errors
@ -105,9 +105,9 @@ def main(opts):
print("Must provide exactly one of -d or -u")
return 1
print("Checking logs...")
WHITELIST_FILE = os.path.join(
ALLOW_LIST_FILE = os.path.join(
os.path.abspath(os.path.dirname(os.path.dirname(__file__))),
"etc", "whitelist.yaml")
"etc", "allow-list.yaml")
file_matcher = re.compile(r".*screen-([\w-]+)\.log")
files = []
@ -132,17 +132,17 @@ def main(opts):
if m:
urls_to_process.append((m.group(1), u))
whitelists = {}
with open(WHITELIST_FILE) as stream:
allow_lists = {}
with open(ALLOW_LIST_FILE) as stream:
loaded = yaml.safe_load(stream)
if loaded:
for (name, l) in six.iteritems(loaded):
for w in l:
assert 'module' in w, 'no module in %s' % name
assert 'message' in w, 'no message in %s' % name
whitelists = loaded
allow_lists = loaded
logs_with_errors = process_files(files_to_process, urls_to_process,
whitelists)
allow_lists)
failed = False
if logs_with_errors:
@ -164,14 +164,14 @@ def main(opts):
usage = """
Find non-white-listed log errors in log files from a devstack-gate run.
Find non-allow-listed log errors in log files from a devstack-gate run.
Log files will be searched for ERROR or CRITICAL messages. If any
error messages do not match any of the whitelist entries contained in
etc/whitelist.yaml, those messages will be printed to the console and
error messages do not match any of the allow-list entries contained in
etc/allow-list.yaml, those messages will be printed to the console and
failure will be returned. A file directory containing logs or a url to the
log files of an OpenStack gate job can be provided.
The whitelist yaml looks like:
The allow-list yaml looks like:
log-name:
- module: "a.b.c"
@ -179,7 +179,7 @@ log-name:
- module: "a.b.c"
message: "regexp"
repeated for each log file with a whitelist.
repeated for each log file with an allow-list.
"""
parser = argparse.ArgumentParser(description=usage)

16
tools/generate-tempest-plugins-list.py
View File

@ -32,9 +32,9 @@ from urllib3.util import retry
# List of projects having tempest plugin stale or unmaintained for a long time
# (6 months or more)
# TODO(masayukig): Some of these can be removed from BLACKLIST in the future
# when the patches are merged.
BLACKLIST = [
# TODO(masayukig): Some of these can be removed from NON_ACTIVE_LIST in the
# future when the patches are merged.
NON_ACTIVE_LIST = [
'x/gce-api', # It looks gce-api doesn't support python3 yet.
'x/glare', # To avoid sanity-job failure
'x/group-based-policy', # It looks this doesn't support python3 yet.
@ -52,7 +52,7 @@ BLACKLIST = [
'x/tap-as-a-service', # To avoid sanity-job failure
'x/valet', # https://review.opendev.org/#/c/638339/
'x/kingbird', # https://bugs.launchpad.net/kingbird/+bug/1869722
# vmware-nsx is blacklisted since https://review.opendev.org/#/c/736952
# vmware-nsx is excluded since https://review.opendev.org/#/c/736952
'x/vmware-nsx-tempest-plugin',
]
@ -86,10 +86,10 @@ def has_tempest_plugin(proj):
False
if len(sys.argv) > 1 and sys.argv[1] == 'blacklist':
for black_plugin in BLACKLIST:
print(black_plugin)
# We just need BLACKLIST when we use this `blacklist` option.
if len(sys.argv) > 1 and sys.argv[1] == 'nonactivelist':
for non_active_plugin in NON_ACTIVE_LIST:
print(non_active_plugin)
# We just need NON_ACTIVE_LIST when we use this `nonactivelist` option.
# So, this exits here.
sys.exit()

12
tools/generate-tempest-plugins-list.sh
View File

@ -81,17 +81,17 @@ print_plugin_table "${sorted_plugins}"
printf "\n\n"
# Print BLACKLIST
if [[ -r doc/source/data/tempest-blacklisted-plugins-registry.header ]]; then
cat doc/source/data/tempest-blacklisted-plugins-registry.header
# Print NON_ACTIVE_LIST
if [[ -r doc/source/data/tempest-non-active-plugins-registry.header ]]; then
cat doc/source/data/tempest-non-active-plugins-registry.header
fi
blacklist=$(python tools/generate-tempest-plugins-list.py blacklist)
name_col_len=$(echo "${blacklist}" | wc -L)
nonactivelist=$(python tools/generate-tempest-plugins-list.py nonactivelist)
name_col_len=$(echo "${nonactivelist}" | wc -L)
name_col_len=$(( name_col_len + 20 ))
printf "\n\n"
print_plugin_table "${blacklist}"
print_plugin_table "${nonactivelist}"
printf "\n\n"

0
tools/tempest-integrated-gate-compute-blacklist.txt → tools/tempest-integrated-gate-compute-exclude-list.txt
View File

0
tools/tempest-integrated-gate-networking-blacklist.txt → tools/tempest-integrated-gate-networking-exclude-list.txt
View File

0
tools/tempest-integrated-gate-object-storage-blacklist.txt → tools/tempest-integrated-gate-object-storage-exclude-list.txt
View File

0
tools/tempest-integrated-gate-placement-blacklist.txt → tools/tempest-integrated-gate-placement-exclude-list.txt
View File

14
tools/tempest-integrated-gate-storage-blacklist.txt
View File

@ -1,14 +0,0 @@
# This file includes the backlist of tests which need to be
# skipped for Integrated-gate-storage template. Integrated-gate-storage template
# needs to run only Cinder, Glance, Swift and Nova related tests and rest all
# tests will be skipped by below list.
# Skip network, keystone API tests.
tempest.api.network
tempest.api.identity
# Skip network only scenario tests.
tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps
tempest.scenario.test_network_basic_ops.TestNetworkBasicOps
tempest.scenario.test_network_v6.TestGettingAddress
tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps

1
tools/tempest-integrated-gate-storage-blacklist.txt Symbolic link
View File

@ -0,0 +1 @@
tempest-integrated-gate-storage-exclude-list.txt

14
tools/tempest-integrated-gate-storage-exclude-list.txt Normal file
View File

@ -0,0 +1,14 @@
# This file includes the backlist of tests which need to be
# skipped for Integrated-gate-storage template. Integrated-gate-storage template
# needs to run only Cinder, Glance, Swift and Nova related tests and rest all
# tests will be skipped by below list.
# Skip network, keystone API tests.
tempest.api.network
tempest.api.identity
# Skip network only scenario tests.
tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps
tempest.scenario.test_network_basic_ops.TestNetworkBasicOps
tempest.scenario.test_network_v6.TestGettingAddress
tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps

6
tools/tempest-plugin-sanity.sh
View File

@ -44,7 +44,7 @@ set -ex
# retrieve a list of projects having tempest plugins
PROJECT_LIST="$(python tools/generate-tempest-plugins-list.py)"
BLACKLIST="$(python tools/generate-tempest-plugins-list.py blacklist)"
NON_ACTIVE_LIST="$(python tools/generate-tempest-plugins-list.py nonactivelist)"
# Function to clone project using zuul-cloner or from git
function clone_project {
@ -117,8 +117,8 @@ passed_plugin=''
failed_plugin=''
# Perform sanity on all tempest plugin projects
for project in $PROJECT_LIST; do
# Remove blacklisted tempest plugins
if ! [[ `echo $BLACKLIST | grep -c $project ` -gt 0 ]]; then
# Remove non-active tempest plugins
if ! [[ `echo $NON_ACTIVE_LIST | grep -c $project ` -gt 0 ]]; then
plugin_sanity_check $project && passed_plugin+=", $project" || \
failed_plugin+="$project, " > $SANITY_DIR/$project.txt
fi

48
tox.ini
View File

@ -1,6 +1,6 @@
[tox]
envlist = pep8,py36,py38,bashate,pip-check-reqs
minversion = 3.1.1
minversion = 3.18.0
skipsdist = True
ignore_basepython_conflict = True
@ -26,7 +26,7 @@ setenv =
passenv = OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_TEST_TIMEOUT OS_TEST_LOCK_PATH TEMPEST_CONFIG TEMPEST_CONFIG_DIR http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY ZUUL_CACHE_DIR REQUIREMENTS_PIP_LOCATION GENERATE_TEMPEST_PLUGIN_LIST
usedevelop = True
install_command = pip install {opts} {packages}
whitelist_externals = *
allowlist_externals = *
deps =
-c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
-r{toxinidir}/requirements.txt
@ -108,7 +108,7 @@ setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag:
# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610
# FIXME: We can replace it with the `--black-regex` option to exclude tests now.
# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now.
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' {posargs}
@ -132,11 +132,11 @@ basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in blacklist file:
# tests listed in exclude-list file:
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --blacklist_file ./tools/tempest-integrated-gate-networking-blacklist.txt {posargs}
tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --blacklist_file ./tools/tempest-integrated-gate-networking-blacklist.txt {posargs}
tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-networking-exclude-list.txt {posargs}
tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exlude-list ./tools/tempest-integrated-gate-networking-exclude-list.txt {posargs}
[testenv:integrated-compute]
envdir = .tox/tempest
@ -145,11 +145,11 @@ basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in blacklist file:
# tests listed in exclude-list file:
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --blacklist_file ./tools/tempest-integrated-gate-compute-blacklist.txt {posargs}
tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --blacklist_file ./tools/tempest-integrated-gate-compute-blacklist.txt {posargs}
tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-compute-exclude-list.txt {posargs}
tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exclude-list ./tools/tempest-integrated-gate-compute-exclude-list.txt {posargs}
[testenv:integrated-placement]
envdir = .tox/tempest
@ -158,11 +158,11 @@ basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in blacklist file:
# tests listed in exclude-list file:
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --blacklist_file ./tools/tempest-integrated-gate-placement-blacklist.txt {posargs}
tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --blacklist_file ./tools/tempest-integrated-gate-placement-blacklist.txt {posargs}
tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-placement-exclude-list.txt {posargs}
tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exclude-list ./tools/tempest-integrated-gate-placement-exclude-list.txt {posargs}
[testenv:integrated-storage]
envdir = .tox/tempest
@ -171,11 +171,11 @@ basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in blacklist file:
# tests listed in exclude-list file:
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --blacklist_file ./tools/tempest-integrated-gate-storage-blacklist.txt {posargs}
tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --blacklist_file ./tools/tempest-integrated-gate-storage-blacklist.txt {posargs}
tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-storage-exclude-list.txt {posargs}
tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exclude-list ./tools/tempest-integrated-gate-storage-exclude-list.txt {posargs}
[testenv:integrated-object-storage]
envdir = .tox/tempest
@ -184,11 +184,11 @@ basepython = {[tempestenv]basepython}
setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag and
# tests listed in blacklist file:
# tests listed in exclude-list file:
commands =
find . -type f -name "*.pyc" -delete
tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --blacklist_file ./tools/tempest-integrated-gate-object-storage-blacklist.txt {posargs}
tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --blacklist_file ./tools/tempest-integrated-gate-object-storage-blacklist.txt {posargs}
tempest run --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' --exclude-list ./tools/tempest-integrated-gate-object-storage-exclude-list.txt {posargs}
tempest run --combine --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' --exclude-list ./tools/tempest-integrated-gate-object-storage-exclude-list.txt {posargs}
[testenv:full-serial]
envdir = .tox/tempest
@ -198,7 +198,7 @@ setenv = {[tempestenv]setenv}
deps = {[tempestenv]deps}
# The regex below is used to select which tests to run and exclude the slow tag:
# See the testrepository bug: https://bugs.launchpad.net/testrepository/+bug/1208610
# FIXME: We can replace it with the `--black-regex` option to exclude tests now.
# FIXME: We can replace it with the `--exclude-regex` option to exclude tests now.
commands =
find . -type f -name "*.pyc" -delete
tempest run --serial --regex '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario))' {posargs}
@ -290,12 +290,12 @@ commands =
sphinx-apidoc -f -o doc/source/tests/volume tempest/api/volume
rm -rf doc/build
sphinx-build -W -b html doc/source doc/build/html
whitelist_externals =
allowlist_externals =
rm
[testenv:pdf-docs]
deps = {[testenv:docs]deps}
whitelist_externals =
allowlist_externals =
rm
make
commands =
@ -369,7 +369,7 @@ commands =
rm -rf releasenotes/build
sphinx-build -a -E -W -d releasenotes/build/doctrees \
-b html releasenotes/source releasenotes/build/html
whitelist_externals = rm
allowlist_externals = rm
[testenv:bashate]
# if you want to test out some changes you have made to bashate
@ -377,7 +377,7 @@ whitelist_externals = rm
# modified bashate tree
deps =
{env:BASHATE_INSTALL_PATH:bashate}
whitelist_externals = bash
allowlist_externals = bash
commands = bash -c "find {toxinidir}/tools \
-not \( -type d -name .?\* -prune \) \
-type f \
@ -406,6 +406,6 @@ commands = bindep test
[testenv:plugin-sanity-check]
# perform tempest plugin sanity
whitelist_externals = bash
allowlist_externals = bash
commands =
bash tools/tempest-plugin-sanity.sh