diff --git a/doc/source/listyaml/listyaml.rst b/doc/source/listyaml/listyaml.rst index ba8f5f1..fd08093 100644 --- a/doc/source/listyaml/listyaml.rst +++ b/doc/source/listyaml/listyaml.rst @@ -13,19 +13,20 @@ format:: $ tempest-skip list-skipped yaml --file tempest_skip.yml - 2. ``--release``, ``--deployment`` and ``--job`` are the optional - parameters - list all the tests within a specific release, deployment - or a specific job:: + 2. ``--release``, ``--deployment``, ``--installer`` and ``--job`` are the + optional parameters - list all the tests within a specific release, + deployment or a specific job:: $ tempest-skip list-skipped --file tempest_skip.yml --release train $ tempest-skip list-skipped --file tempest_skip.yml --job job1 $ tempest-skip list-skipped --file tempest_skip.yml --release train --job job1 $ tempest-skip list-skipped --file tempest_skip.yml --deployment undercloud + $ tempest-skip list-skipped --file tempest_skip.yml --installer tripleo This will return any tests that match the job, as well as tests that doesn't have any job configured. This is required when you configure your zuul jobs to -always parse the --job option. In this scenario, if a job2 is parsed, and there -is no test with job2, it would return zero tests to be skipped, which is not -the intent. The test with no job defined, means, skip everywhere, if you +always parse the ``--job`` option. In this scenario, if a job2 is parsed, and +there is no test with job2, it would return zero tests to be skipped, which is +not the intent. The test with no job defined, means, skip everywhere, if you define the job in the test yaml file, it means, skip all the tests that doesn't have a job defined, plus this test. diff --git a/roles/list_skipped/library/list_skipped.py b/roles/list_skipped/library/list_skipped.py index 2d188a9..a587fed 100644 --- a/roles/list_skipped/library/list_skipped.py +++ b/roles/list_skipped/library/list_skipped.py @@ -61,6 +61,12 @@ options: set to 'overcloud' required: False type: str + installer: + description: + - Type of installer. Right now it's just a string, but in the future + might be specific keys + required: False + type: str ''' @@ -71,6 +77,7 @@ EXAMPLES = ''' job: tripleo-ci-centos-8-standalone release: master deployment: 'overcloud' + installer: 'tripleo' ''' @@ -91,7 +98,8 @@ def run_module(): yaml_file=dict(type='str', required=True), job=dict(type='str', required=False, default=None), release=dict(type='str', required=False, default='master'), - deployment=dict(type='str', required=False, default='overcloud') + deployment=dict(type='str', required=False, default='overcloud'), + installer=dict(type='str', required=False, default='tripleo') ) result = dict( @@ -112,6 +120,7 @@ def run_module(): parser.release = module.params['release'] parser.job = module.params['job'] parser.deployment = module.params['deployment'] + parser.installer = module.params['installer'] tests = cmd.take_action(parser) skipped_tests = [test[0] for test in tests[1]] diff --git a/roles/list_skipped/tasks/main.yaml b/roles/list_skipped/tasks/main.yaml index 1b5eb01..0b83dae 100644 --- a/roles/list_skipped/tasks/main.yaml +++ b/roles/list_skipped/tasks/main.yaml @@ -5,4 +5,5 @@ release: "{{ list_skipped_release | default(omit) }}" job: "{{ list_skipped_job | default(omit) }}" deployment: "{{ list_skipped_deployment | default(omit) }}" + installer: "{{ list_skipped_installer | default(omit) }}" register: tempest_skip_register diff --git a/tempest_skip/list_yaml.py b/tempest_skip/list_yaml.py index eb7ae41..2d0e4b7 100644 --- a/tempest_skip/list_yaml.py +++ b/tempest_skip/list_yaml.py @@ -42,37 +42,28 @@ class ListSkippedYaml(Lister): tests = yaml_file.get('known_failures', []) if parsed_args.job: - self.parsed_job = parsed_args.job - tests = list(filter(self._filter_jobs, tests)) + tests = [test for test in tests + if (not test.get('jobs', []) or ( + parsed_args.job in test.get('jobs')))] if parsed_args.release: - new_tests = [] - for test in tests: - for release in test.get('releases', []): - if release.get('name') == parsed_args.release: - new_tests.append(test) - tests = new_tests + tests = [test for test in tests + if [release for release in test.get('releases', []) + if release['name'] == parsed_args.release]] + + tests = [test for test in tests + if [release for release in test.get('releases', []) + if parsed_args.installer in + release.get('installers', ['tripleo', 'osp'])]] if parsed_args.deployment: - self.parsed_deployment = parsed_args.deployment - tests = list(filter(self._filter_deployment, tests)) + tests = [test for test in tests + if (not test.get('deployment', []) or ( + parsed_args.deployment in + test.get('deployment')))] return tests return [] - def _filter_deployment(self, test): - if not test.get('deployment', []): - return True - if self.parsed_deployment in test.get('deployment'): - return True - return False - - def _filter_jobs(self, test): - if not test.get('jobs', []): - return True - if self.parsed_job in test.get('jobs'): - return True - return False - def get_parser(self, prog_name): parser = super(ListSkippedYaml, self).get_parser(prog_name) parser.add_argument('--file', dest='file', @@ -91,4 +82,9 @@ class ListSkippedYaml(Lister): parser.add_argument('--deployment', dest='deployment', help='List the tests to be skipped in the ' 'given deployment') + parser.add_argument('--installer', dest='installer', + default=None, help='Tests to be skipped for a ' + 'particular installer. Use ' + 'tripleo for upstream, and osp' + ' for downstream') return parser diff --git a/tempest_skip/tests/test_list_yaml.py b/tempest_skip/tests/test_list_yaml.py index 5841c77..238f28f 100644 --- a/tempest_skip/tests/test_list_yaml.py +++ b/tempest_skip/tests/test_list_yaml.py @@ -48,6 +48,31 @@ class TestListSkippedYaml(base.TestCase): lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166' jobs: - 'job1' + - test: 'tempest_skip.tests.test_list_yaml_4' + deployment: + - 'overcloud' + releases: + - name: 'train' + reason: 'Test failing on train release' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166' + installers: + - 'osp' + jobs: + - 'job3' + - test: 'tempest_skip.tests.test_list_yaml_5' + deployment: + - 'overcloud' + releases: + - name: 'train' + reason: 'Test failing on train release' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166' + installers: + - 'osp' + - 'tripleo' + - name: 'wallaby' + reason: 'Test failing on train release' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166' + jobs: [] """ self.path = self.write_yaml_file(self.list_file) @@ -57,13 +82,15 @@ class TestListSkippedYaml(base.TestCase): self.parser.release = None self.parser.deployment = None self.parser.job = None + self.parser.installer = 'tripleo' def test_list_yaml(self): cmd_result = self.cmd.take_action(self.parser) exptected = [('tempest_skip.tests.test_list_yaml',), ('tempest_skip.tests.test_list_yaml_2',), - ('tempest_skip.tests.test_list_yaml_3',)] + ('tempest_skip.tests.test_list_yaml_3',), + ('tempest_skip.tests.test_list_yaml_5',)] list_tests = [test for test in cmd_result[1]] self.assertEqual(exptected, list_tests) @@ -72,7 +99,8 @@ class TestListSkippedYaml(base.TestCase): self.parser.job = None cmd_result = self.cmd.take_action(self.parser) - exptected = [('tempest_skip.tests.test_list_yaml_3',)] + exptected = [('tempest_skip.tests.test_list_yaml_3',), + ('tempest_skip.tests.test_list_yaml_5',)] list_tests = [test for test in cmd_result[1]] self.assertEqual(exptected, list_tests) @@ -102,6 +130,35 @@ class TestListSkippedYaml(base.TestCase): list_tests = [test for test in cmd_result[1]] self.assertEqual(expected, list_tests) + def test_list_yaml_with_installer(self): + self.parser.installer = 'osp' + self.parser.release = 'master' + cmd_result = self.cmd.take_action(self.parser) + expected = [('tempest_skip.tests.test_list_yaml',), + ('tempest_skip.tests.test_list_yaml_2',)] + list_tests = [test for test in cmd_result[1]] + self.assertEqual(expected, list_tests) + + def test_list_yaml_with_installer_and_deployment(self): + self.parser.installer = 'osp' + self.parser.deployment = 'overcloud' + cmd_result = self.cmd.take_action(self.parser) + expected = [('tempest_skip.tests.test_list_yaml',), + ('tempest_skip.tests.test_list_yaml_2',), + ('tempest_skip.tests.test_list_yaml_3',), + ('tempest_skip.tests.test_list_yaml_4',), + ('tempest_skip.tests.test_list_yaml_5',)] + list_tests = [test for test in cmd_result[1]] + self.assertEqual(expected, list_tests) + + def test_list_yaml_with_installer_and_invalid_deployment(self): + self.parser.installer = 'osp' + self.parser.deployment = 'underclouds' + cmd_result = self.cmd.take_action(self.parser) + expected = [] + list_tests = [test for test in cmd_result[1]] + self.assertEqual(expected, list_tests) + def test_list_yaml_with_release_not_found(self): self.parser.release = 'not-found' self.parser.job = None @@ -116,7 +173,8 @@ class TestListSkippedYaml(base.TestCase): cmd_result = self.cmd.take_action(self.parser) exptected = [('tempest_skip.tests.test_list_yaml',), ('tempest_skip.tests.test_list_yaml_2',), - ('tempest_skip.tests.test_list_yaml_3',)] + ('tempest_skip.tests.test_list_yaml_3',), + ('tempest_skip.tests.test_list_yaml_5',)] list_tests = [test for test in cmd_result[1]] self.assertEqual(exptected, list_tests) @@ -124,7 +182,8 @@ class TestListSkippedYaml(base.TestCase): self.parser.job = 'job2' cmd_result = self.cmd.take_action(self.parser) exptected = [('tempest_skip.tests.test_list_yaml',), - ('tempest_skip.tests.test_list_yaml_2',)] + ('tempest_skip.tests.test_list_yaml_2',), + ('tempest_skip.tests.test_list_yaml_5',)] list_tests = [test for test in cmd_result[1]] self.assertEqual(exptected, list_tests) @@ -132,7 +191,8 @@ class TestListSkippedYaml(base.TestCase): self.parser.job = 'job1' self.parser.release = 'train' cmd_result = self.cmd.take_action(self.parser) - exptected = [('tempest_skip.tests.test_list_yaml_3',)] + exptected = [('tempest_skip.tests.test_list_yaml_3',), + ('tempest_skip.tests.test_list_yaml_5',)] list_tests = [test for test in cmd_result[1]] self.assertEqual(exptected, list_tests) diff --git a/tempest_skip/tests/test_validate.py b/tempest_skip/tests/test_validate.py index 660f095..36d5aca 100644 --- a/tempest_skip/tests/test_validate.py +++ b/tempest_skip/tests/test_validate.py @@ -110,6 +110,9 @@ class TestValidateSkipped(TestValidate): - name: 'master' lp: 'https://launchpad.net/bugs/1' reason: 'Test with launchpad' + installers: + - 'tripleo' + - 'osp' - name: 'train' bz: 'https://bugzilla.redhat.com/1' reason: 'Test with bugzilla' diff --git a/tempest_skip/validate.py b/tempest_skip/validate.py index 46df358..b7ffe79 100644 --- a/tempest_skip/validate.py +++ b/tempest_skip/validate.py @@ -38,7 +38,8 @@ class Validate(Command): v.Required('name'): str, v.Required(v.SomeOf( validators=[v.Any('lp', 'bz')], min_valid=1)): v.Url(), - v.Required('reason'): str + v.Required('reason'): str, + v.Optional('installers'): [str] }) ], v.Optional('jobs'): [str]