openstack/openstack-tempest-skiplist
Code Issues Proposed changes

Add installer key to releases on skiplist

Browse Source 
Due the fact that one release might have different installers and
recently we were facing problems where add a test on the skiplist in
upstream jobs was also affecting downstream jobs, the best solution
right now is to add the key installers on the release for the test.
The default remains being 'tripleo' and 'osp', so we don't need to edit
all the tests to add the installers option. However, if you explicitly
set the installer, and you want it to be skipped in both tripleo and
other, you must also add the tripleo to the list.

Change-Id: I709a826f75e64ab2a7f741891e0a3cc4aa6edf32
This commit is contained in:
Arx Cruz 2021-05-04 21:29:01 +02:00
parent 19332d036e
commit 85a392c55f
7 changed files with 108 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
doc/source/listyaml/listyaml.rst
View File

@ -13,19 +13,20 @@ format::
$ tempest-skip list-skipped yaml --file tempest_skip.yml
2. ``--release``, ``--deployment`` and ``--job`` are the optional
parameters - list all the tests within a specific release, deployment
or a specific job::
2. ``--release``, ``--deployment``, ``--installer`` and ``--job`` are the
optional parameters - list all the tests within a specific release,
deployment or a specific job::
$ tempest-skip list-skipped --file tempest_skip.yml --release train
$ tempest-skip list-skipped --file tempest_skip.yml --job job1
$ tempest-skip list-skipped --file tempest_skip.yml --release train --job job1
$ tempest-skip list-skipped --file tempest_skip.yml --deployment undercloud
$ tempest-skip list-skipped --file tempest_skip.yml --installer tripleo
This will return any tests that match the job, as well as tests that doesn't
have any job configured. This is required when you configure your zuul jobs to
always parse the --job option. In this scenario, if a job2 is parsed, and there
is no test with job2, it would return zero tests to be skipped, which is not
the intent. The test with no job defined, means, skip everywhere, if you
always parse the ``--job`` option. In this scenario, if a job2 is parsed, and
there is no test with job2, it would return zero tests to be skipped, which is
not the intent. The test with no job defined, means, skip everywhere, if you
define the job in the test yaml file, it means, skip all the tests that doesn't
have a job defined, plus this test.

11
roles/list_skipped/library/list_skipped.py
View File

@ -61,6 +61,12 @@ options:
set to 'overcloud'
required: False
type: str
installer:
description:
- Type of installer. Right now it's just a string, but in the future
might be specific keys
required: False
type: str
'''
@ -71,6 +77,7 @@ EXAMPLES = '''
job: tripleo-ci-centos-8-standalone
release: master
deployment: 'overcloud'
installer: 'tripleo'
'''
@ -91,7 +98,8 @@ def run_module():
yaml_file=dict(type='str', required=True),
job=dict(type='str', required=False, default=None),
release=dict(type='str', required=False, default='master'),
deployment=dict(type='str', required=False, default='overcloud')
deployment=dict(type='str', required=False, default='overcloud'),
installer=dict(type='str', required=False, default='tripleo')
)
result = dict(
@ -112,6 +120,7 @@ def run_module():
parser.release = module.params['release']
parser.job = module.params['job']
parser.deployment = module.params['deployment']
parser.installer = module.params['installer']
tests = cmd.take_action(parser)
skipped_tests = [test[0] for test in tests[1]]

1
roles/list_skipped/tasks/main.yaml
View File

@ -5,4 +5,5 @@
release: "{{ list_skipped_release | default(omit) }}"
job: "{{ list_skipped_job | default(omit) }}"
deployment: "{{ list_skipped_deployment | default(omit) }}"
installer: "{{ list_skipped_installer | default(omit) }}"
register: tempest_skip_register

44
tempest_skip/list_yaml.py
View File

@ -42,37 +42,28 @@ class ListSkippedYaml(Lister):
tests = yaml_file.get('known_failures', [])
if parsed_args.job:
self.parsed_job = parsed_args.job
tests = list(filter(self._filter_jobs, tests))
tests = [test for test in tests
if (not test.get('jobs', []) or (
parsed_args.job in test.get('jobs')))]
if parsed_args.release:
new_tests = []
for test in tests:
for release in test.get('releases', []):
if release.get('name') == parsed_args.release:
new_tests.append(test)
tests = new_tests
tests = [test for test in tests
if [release for release in test.get('releases', [])
if release['name'] == parsed_args.release]]
tests = [test for test in tests
if [release for release in test.get('releases', [])
if parsed_args.installer in
release.get('installers', ['tripleo', 'osp'])]]
if parsed_args.deployment:
self.parsed_deployment = parsed_args.deployment
tests = list(filter(self._filter_deployment, tests))
tests = [test for test in tests
if (not test.get('deployment', []) or (
parsed_args.deployment in
test.get('deployment')))]
return tests
return []
def _filter_deployment(self, test):
if not test.get('deployment', []):
return True
if self.parsed_deployment in test.get('deployment'):
return True
return False
def _filter_jobs(self, test):
if not test.get('jobs', []):
return True
if self.parsed_job in test.get('jobs'):
return True
return False
def get_parser(self, prog_name):
parser = super(ListSkippedYaml, self).get_parser(prog_name)
parser.add_argument('--file', dest='file',
@ -91,4 +82,9 @@ class ListSkippedYaml(Lister):
parser.add_argument('--deployment', dest='deployment',
help='List the tests to be skipped in the '
'given deployment')
parser.add_argument('--installer', dest='installer',
default=None, help='Tests to be skipped for a '
'particular installer. Use '
'tripleo for upstream, and osp'
' for downstream')
return parser

70
tempest_skip/tests/test_list_yaml.py
View File

@ -48,6 +48,31 @@ class TestListSkippedYaml(base.TestCase):
lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166'
jobs:
- 'job1'
- test: 'tempest_skip.tests.test_list_yaml_4'
deployment:
- 'overcloud'
releases:
- name: 'train'
reason: 'Test failing on train release'
lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166'
installers:
- 'osp'
jobs:
- 'job3'
- test: 'tempest_skip.tests.test_list_yaml_5'
deployment:
- 'overcloud'
releases:
- name: 'train'
reason: 'Test failing on train release'
lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166'
installers:
- 'osp'
- 'tripleo'
- name: 'wallaby'
reason: 'Test failing on train release'
lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166'
jobs: []
"""
self.path = self.write_yaml_file(self.list_file)
@ -57,13 +82,15 @@ class TestListSkippedYaml(base.TestCase):
self.parser.release = None
self.parser.deployment = None
self.parser.job = None
self.parser.installer = 'tripleo'
def test_list_yaml(self):
cmd_result = self.cmd.take_action(self.parser)
exptected = [('tempest_skip.tests.test_list_yaml',),
('tempest_skip.tests.test_list_yaml_2',),
('tempest_skip.tests.test_list_yaml_3',)]
('tempest_skip.tests.test_list_yaml_3',),
('tempest_skip.tests.test_list_yaml_5',)]
list_tests = [test for test in cmd_result[1]]
self.assertEqual(exptected, list_tests)
@ -72,7 +99,8 @@ class TestListSkippedYaml(base.TestCase):
self.parser.job = None
cmd_result = self.cmd.take_action(self.parser)
exptected = [('tempest_skip.tests.test_list_yaml_3',)]
exptected = [('tempest_skip.tests.test_list_yaml_3',),
('tempest_skip.tests.test_list_yaml_5',)]
list_tests = [test for test in cmd_result[1]]
self.assertEqual(exptected, list_tests)
@ -102,6 +130,35 @@ class TestListSkippedYaml(base.TestCase):
list_tests = [test for test in cmd_result[1]]
self.assertEqual(expected, list_tests)
def test_list_yaml_with_installer(self):
self.parser.installer = 'osp'
self.parser.release = 'master'
cmd_result = self.cmd.take_action(self.parser)
expected = [('tempest_skip.tests.test_list_yaml',),
('tempest_skip.tests.test_list_yaml_2',)]
list_tests = [test for test in cmd_result[1]]
self.assertEqual(expected, list_tests)
def test_list_yaml_with_installer_and_deployment(self):
self.parser.installer = 'osp'
self.parser.deployment = 'overcloud'
cmd_result = self.cmd.take_action(self.parser)
expected = [('tempest_skip.tests.test_list_yaml',),
('tempest_skip.tests.test_list_yaml_2',),
('tempest_skip.tests.test_list_yaml_3',),
('tempest_skip.tests.test_list_yaml_4',),
('tempest_skip.tests.test_list_yaml_5',)]
list_tests = [test for test in cmd_result[1]]
self.assertEqual(expected, list_tests)
def test_list_yaml_with_installer_and_invalid_deployment(self):
self.parser.installer = 'osp'
self.parser.deployment = 'underclouds'
cmd_result = self.cmd.take_action(self.parser)
expected = []
list_tests = [test for test in cmd_result[1]]
self.assertEqual(expected, list_tests)
def test_list_yaml_with_release_not_found(self):
self.parser.release = 'not-found'
self.parser.job = None
@ -116,7 +173,8 @@ class TestListSkippedYaml(base.TestCase):
cmd_result = self.cmd.take_action(self.parser)
exptected = [('tempest_skip.tests.test_list_yaml',),
('tempest_skip.tests.test_list_yaml_2',),
('tempest_skip.tests.test_list_yaml_3',)]
('tempest_skip.tests.test_list_yaml_3',),
('tempest_skip.tests.test_list_yaml_5',)]
list_tests = [test for test in cmd_result[1]]
self.assertEqual(exptected, list_tests)
@ -124,7 +182,8 @@ class TestListSkippedYaml(base.TestCase):
self.parser.job = 'job2'
cmd_result = self.cmd.take_action(self.parser)
exptected = [('tempest_skip.tests.test_list_yaml',),
('tempest_skip.tests.test_list_yaml_2',)]
('tempest_skip.tests.test_list_yaml_2',),
('tempest_skip.tests.test_list_yaml_5',)]
list_tests = [test for test in cmd_result[1]]
self.assertEqual(exptected, list_tests)
@ -132,7 +191,8 @@ class TestListSkippedYaml(base.TestCase):
self.parser.job = 'job1'
self.parser.release = 'train'
cmd_result = self.cmd.take_action(self.parser)
exptected = [('tempest_skip.tests.test_list_yaml_3',)]
exptected = [('tempest_skip.tests.test_list_yaml_3',),
('tempest_skip.tests.test_list_yaml_5',)]
list_tests = [test for test in cmd_result[1]]
self.assertEqual(exptected, list_tests)

3
tempest_skip/tests/test_validate.py
View File

@ -110,6 +110,9 @@ class TestValidateSkipped(TestValidate):
- name: 'master'
lp: 'https://launchpad.net/bugs/1'
reason: 'Test with launchpad'
installers:
- 'tripleo'
- 'osp'
- name: 'train'
bz: 'https://bugzilla.redhat.com/1'
reason: 'Test with bugzilla'

3
tempest_skip/validate.py
View File

@ -38,7 +38,8 @@ class Validate(Command):
v.Required('name'): str,
v.Required(v.SomeOf(
validators=[v.Any('lp', 'bz')], min_valid=1)): v.Url(),
v.Required('reason'): str
v.Required('reason'): str,
v.Optional('installers'): [str]
})
],
v.Optional('jobs'): [str]