From 85a392c55f55bd83a370c677c3174d567f288b1e Mon Sep 17 00:00:00 2001 From: Arx Cruz Date: Tue, 4 May 2021 21:29:01 +0200 Subject: [PATCH] Add installer key to releases on skiplist Due the fact that one release might have different installers and recently we were facing problems where add a test on the skiplist in upstream jobs was also affecting downstream jobs, the best solution right now is to add the key installers on the release for the test. The default remains being 'tripleo' and 'osp', so we don't need to edit all the tests to add the installers option. However, if you explicitly set the installer, and you want it to be skipped in both tripleo and other, you must also add the tripleo to the list. Change-Id: I709a826f75e64ab2a7f741891e0a3cc4aa6edf32 --- doc/source/listyaml/listyaml.rst | 13 ++-- roles/list_skipped/library/list_skipped.py | 11 +++- roles/list_skipped/tasks/main.yaml | 1 + tempest_skip/list_yaml.py | 44 +++++++------- tempest_skip/tests/test_list_yaml.py | 70 ++++++++++++++++++++-- tempest_skip/tests/test_validate.py | 3 + tempest_skip/validate.py | 3 +- 7 files changed, 108 insertions(+), 37 deletions(-) diff --git a/doc/source/listyaml/listyaml.rst b/doc/source/listyaml/listyaml.rst index ba8f5f1..fd08093 100644 --- a/doc/source/listyaml/listyaml.rst +++ b/doc/source/listyaml/listyaml.rst @@ -13,19 +13,20 @@ format:: $ tempest-skip list-skipped yaml --file tempest_skip.yml - 2. ``--release``, ``--deployment`` and ``--job`` are the optional - parameters - list all the tests within a specific release, deployment - or a specific job:: + 2. ``--release``, ``--deployment``, ``--installer`` and ``--job`` are the + optional parameters - list all the tests within a specific release, + deployment or a specific job:: $ tempest-skip list-skipped --file tempest_skip.yml --release train $ tempest-skip list-skipped --file tempest_skip.yml --job job1 $ tempest-skip list-skipped --file tempest_skip.yml --release train --job job1 $ tempest-skip list-skipped --file tempest_skip.yml --deployment undercloud + $ tempest-skip list-skipped --file tempest_skip.yml --installer tripleo This will return any tests that match the job, as well as tests that doesn't have any job configured. This is required when you configure your zuul jobs to -always parse the --job option. In this scenario, if a job2 is parsed, and there -is no test with job2, it would return zero tests to be skipped, which is not -the intent. The test with no job defined, means, skip everywhere, if you +always parse the ``--job`` option. In this scenario, if a job2 is parsed, and +there is no test with job2, it would return zero tests to be skipped, which is +not the intent. The test with no job defined, means, skip everywhere, if you define the job in the test yaml file, it means, skip all the tests that doesn't have a job defined, plus this test. diff --git a/roles/list_skipped/library/list_skipped.py b/roles/list_skipped/library/list_skipped.py index 2d188a9..a587fed 100644 --- a/roles/list_skipped/library/list_skipped.py +++ b/roles/list_skipped/library/list_skipped.py @@ -61,6 +61,12 @@ options: set to 'overcloud' required: False type: str + installer: + description: + - Type of installer. Right now it's just a string, but in the future + might be specific keys + required: False + type: str ''' @@ -71,6 +77,7 @@ EXAMPLES = ''' job: tripleo-ci-centos-8-standalone release: master deployment: 'overcloud' + installer: 'tripleo' ''' @@ -91,7 +98,8 @@ def run_module(): yaml_file=dict(type='str', required=True), job=dict(type='str', required=False, default=None), release=dict(type='str', required=False, default='master'), - deployment=dict(type='str', required=False, default='overcloud') + deployment=dict(type='str', required=False, default='overcloud'), + installer=dict(type='str', required=False, default='tripleo') ) result = dict( @@ -112,6 +120,7 @@ def run_module(): parser.release = module.params['release'] parser.job = module.params['job'] parser.deployment = module.params['deployment'] + parser.installer = module.params['installer'] tests = cmd.take_action(parser) skipped_tests = [test[0] for test in tests[1]] diff --git a/roles/list_skipped/tasks/main.yaml b/roles/list_skipped/tasks/main.yaml index 1b5eb01..0b83dae 100644 --- a/roles/list_skipped/tasks/main.yaml +++ b/roles/list_skipped/tasks/main.yaml @@ -5,4 +5,5 @@ release: "{{ list_skipped_release | default(omit) }}" job: "{{ list_skipped_job | default(omit) }}" deployment: "{{ list_skipped_deployment | default(omit) }}" + installer: "{{ list_skipped_installer | default(omit) }}" register: tempest_skip_register diff --git a/tempest_skip/list_yaml.py b/tempest_skip/list_yaml.py index eb7ae41..2d0e4b7 100644 --- a/tempest_skip/list_yaml.py +++ b/tempest_skip/list_yaml.py @@ -42,37 +42,28 @@ class ListSkippedYaml(Lister): tests = yaml_file.get('known_failures', []) if parsed_args.job: - self.parsed_job = parsed_args.job - tests = list(filter(self._filter_jobs, tests)) + tests = [test for test in tests + if (not test.get('jobs', []) or ( + parsed_args.job in test.get('jobs')))] if parsed_args.release: - new_tests = [] - for test in tests: - for release in test.get('releases', []): - if release.get('name') == parsed_args.release: - new_tests.append(test) - tests = new_tests + tests = [test for test in tests + if [release for release in test.get('releases', []) + if release['name'] == parsed_args.release]] + + tests = [test for test in tests + if [release for release in test.get('releases', []) + if parsed_args.installer in + release.get('installers', ['tripleo', 'osp'])]] if parsed_args.deployment: - self.parsed_deployment = parsed_args.deployment - tests = list(filter(self._filter_deployment, tests)) + tests = [test for test in tests + if (not test.get('deployment', []) or ( + parsed_args.deployment in + test.get('deployment')))] return tests return [] - def _filter_deployment(self, test): - if not test.get('deployment', []): - return True - if self.parsed_deployment in test.get('deployment'): - return True - return False - - def _filter_jobs(self, test): - if not test.get('jobs', []): - return True - if self.parsed_job in test.get('jobs'): - return True - return False - def get_parser(self, prog_name): parser = super(ListSkippedYaml, self).get_parser(prog_name) parser.add_argument('--file', dest='file', @@ -91,4 +82,9 @@ class ListSkippedYaml(Lister): parser.add_argument('--deployment', dest='deployment', help='List the tests to be skipped in the ' 'given deployment') + parser.add_argument('--installer', dest='installer', + default=None, help='Tests to be skipped for a ' + 'particular installer. Use ' + 'tripleo for upstream, and osp' + ' for downstream') return parser diff --git a/tempest_skip/tests/test_list_yaml.py b/tempest_skip/tests/test_list_yaml.py index 5841c77..238f28f 100644 --- a/tempest_skip/tests/test_list_yaml.py +++ b/tempest_skip/tests/test_list_yaml.py @@ -48,6 +48,31 @@ class TestListSkippedYaml(base.TestCase): lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166' jobs: - 'job1' + - test: 'tempest_skip.tests.test_list_yaml_4' + deployment: + - 'overcloud' + releases: + - name: 'train' + reason: 'Test failing on train release' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166' + installers: + - 'osp' + jobs: + - 'job3' + - test: 'tempest_skip.tests.test_list_yaml_5' + deployment: + - 'overcloud' + releases: + - name: 'train' + reason: 'Test failing on train release' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166' + installers: + - 'osp' + - 'tripleo' + - name: 'wallaby' + reason: 'Test failing on train release' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166' + jobs: [] """ self.path = self.write_yaml_file(self.list_file) @@ -57,13 +82,15 @@ class TestListSkippedYaml(base.TestCase): self.parser.release = None self.parser.deployment = None self.parser.job = None + self.parser.installer = 'tripleo' def test_list_yaml(self): cmd_result = self.cmd.take_action(self.parser) exptected = [('tempest_skip.tests.test_list_yaml',), ('tempest_skip.tests.test_list_yaml_2',), - ('tempest_skip.tests.test_list_yaml_3',)] + ('tempest_skip.tests.test_list_yaml_3',), + ('tempest_skip.tests.test_list_yaml_5',)] list_tests = [test for test in cmd_result[1]] self.assertEqual(exptected, list_tests) @@ -72,7 +99,8 @@ class TestListSkippedYaml(base.TestCase): self.parser.job = None cmd_result = self.cmd.take_action(self.parser) - exptected = [('tempest_skip.tests.test_list_yaml_3',)] + exptected = [('tempest_skip.tests.test_list_yaml_3',), + ('tempest_skip.tests.test_list_yaml_5',)] list_tests = [test for test in cmd_result[1]] self.assertEqual(exptected, list_tests) @@ -102,6 +130,35 @@ class TestListSkippedYaml(base.TestCase): list_tests = [test for test in cmd_result[1]] self.assertEqual(expected, list_tests) + def test_list_yaml_with_installer(self): + self.parser.installer = 'osp' + self.parser.release = 'master' + cmd_result = self.cmd.take_action(self.parser) + expected = [('tempest_skip.tests.test_list_yaml',), + ('tempest_skip.tests.test_list_yaml_2',)] + list_tests = [test for test in cmd_result[1]] + self.assertEqual(expected, list_tests) + + def test_list_yaml_with_installer_and_deployment(self): + self.parser.installer = 'osp' + self.parser.deployment = 'overcloud' + cmd_result = self.cmd.take_action(self.parser) + expected = [('tempest_skip.tests.test_list_yaml',), + ('tempest_skip.tests.test_list_yaml_2',), + ('tempest_skip.tests.test_list_yaml_3',), + ('tempest_skip.tests.test_list_yaml_4',), + ('tempest_skip.tests.test_list_yaml_5',)] + list_tests = [test for test in cmd_result[1]] + self.assertEqual(expected, list_tests) + + def test_list_yaml_with_installer_and_invalid_deployment(self): + self.parser.installer = 'osp' + self.parser.deployment = 'underclouds' + cmd_result = self.cmd.take_action(self.parser) + expected = [] + list_tests = [test for test in cmd_result[1]] + self.assertEqual(expected, list_tests) + def test_list_yaml_with_release_not_found(self): self.parser.release = 'not-found' self.parser.job = None @@ -116,7 +173,8 @@ class TestListSkippedYaml(base.TestCase): cmd_result = self.cmd.take_action(self.parser) exptected = [('tempest_skip.tests.test_list_yaml',), ('tempest_skip.tests.test_list_yaml_2',), - ('tempest_skip.tests.test_list_yaml_3',)] + ('tempest_skip.tests.test_list_yaml_3',), + ('tempest_skip.tests.test_list_yaml_5',)] list_tests = [test for test in cmd_result[1]] self.assertEqual(exptected, list_tests) @@ -124,7 +182,8 @@ class TestListSkippedYaml(base.TestCase): self.parser.job = 'job2' cmd_result = self.cmd.take_action(self.parser) exptected = [('tempest_skip.tests.test_list_yaml',), - ('tempest_skip.tests.test_list_yaml_2',)] + ('tempest_skip.tests.test_list_yaml_2',), + ('tempest_skip.tests.test_list_yaml_5',)] list_tests = [test for test in cmd_result[1]] self.assertEqual(exptected, list_tests) @@ -132,7 +191,8 @@ class TestListSkippedYaml(base.TestCase): self.parser.job = 'job1' self.parser.release = 'train' cmd_result = self.cmd.take_action(self.parser) - exptected = [('tempest_skip.tests.test_list_yaml_3',)] + exptected = [('tempest_skip.tests.test_list_yaml_3',), + ('tempest_skip.tests.test_list_yaml_5',)] list_tests = [test for test in cmd_result[1]] self.assertEqual(exptected, list_tests) diff --git a/tempest_skip/tests/test_validate.py b/tempest_skip/tests/test_validate.py index 660f095..36d5aca 100644 --- a/tempest_skip/tests/test_validate.py +++ b/tempest_skip/tests/test_validate.py @@ -110,6 +110,9 @@ class TestValidateSkipped(TestValidate): - name: 'master' lp: 'https://launchpad.net/bugs/1' reason: 'Test with launchpad' + installers: + - 'tripleo' + - 'osp' - name: 'train' bz: 'https://bugzilla.redhat.com/1' reason: 'Test with bugzilla' diff --git a/tempest_skip/validate.py b/tempest_skip/validate.py index 46df358..b7ffe79 100644 --- a/tempest_skip/validate.py +++ b/tempest_skip/validate.py @@ -38,7 +38,8 @@ class Validate(Command): v.Required('name'): str, v.Required(v.SomeOf( validators=[v.Any('lp', 'bz')], min_valid=1)): v.Url(), - v.Required('reason'): str + v.Required('reason'): str, + v.Optional('installers'): [str] }) ], v.Optional('jobs'): [str]