Add installer key to releases on skiplist

Due the fact that one release might have different installers and
recently we were facing problems where add a test on the skiplist in
upstream jobs was also affecting downstream jobs, the best solution
right now is to add the key installers on the release for the test.
The default remains being 'tripleo' and 'osp', so we don't need to edit
all the tests to add the installers option. However, if you explicitly
set the installer, and you want it to be skipped in both tripleo and
other, you must also add the tripleo to the list.

Change-Id: I709a826f75e64ab2a7f741891e0a3cc4aa6edf32

doc/source/listyaml/listyaml.rst

@@ -13,19 +13,20 @@ format::
 
         $ tempest-skip list-skipped yaml --file tempest_skip.yml
 
-    2. ``--release``, ``--deployment`` and ``--job`` are the optional
+    2. ``--release``, ``--deployment``, ``--installer`` and  ``--job`` are the
-       parameters - list all the tests within a specific release, deployment
+       optional parameters - list all the tests within a specific release,
-       or a specific job::
+       deployment or a specific job::
 
        $ tempest-skip list-skipped --file tempest_skip.yml --release train
        $ tempest-skip list-skipped --file tempest_skip.yml --job job1
        $ tempest-skip list-skipped --file tempest_skip.yml --release train --job job1
        $ tempest-skip list-skipped --file tempest_skip.yml --deployment undercloud
+       $ tempest-skip list-skipped --file tempest_skip.yml --installer tripleo
 
 This will return any tests that match the job, as well as tests that doesn't
 have any job configured. This is required when you configure your zuul jobs to
-always parse the --job option. In this scenario, if a job2 is parsed, and there
+always parse the ``--job`` option. In this scenario, if a job2 is parsed, and
-is no test with job2, it would return zero tests to be skipped, which is not
+there is no test with job2, it would return zero tests to be skipped, which is
-the intent. The test with no job defined, means, skip everywhere, if you
+not the intent. The test with no job defined, means, skip everywhere, if you
 define the job in the test yaml file, it means, skip all the tests that doesn't
 have a job defined, plus this test.

roles/list_skipped/library/list_skipped.py

@@ -61,6 +61,12 @@ options:
         set to 'overcloud'
     required: False
     type: str
+  installer:
+    description:
+      - Type of installer. Right now it's just a string, but in the future
+        might be specific keys
+    required: False
+    type: str
 '''
 
 
@@ -71,6 +77,7 @@ EXAMPLES = '''
     job: tripleo-ci-centos-8-standalone
     release: master
     deployment: 'overcloud'
+    installer: 'tripleo'
 '''
 
 
@@ -91,7 +98,8 @@ def run_module():
         yaml_file=dict(type='str', required=True),
         job=dict(type='str', required=False, default=None),
         release=dict(type='str', required=False, default='master'),
-        deployment=dict(type='str', required=False, default='overcloud')
+        deployment=dict(type='str', required=False, default='overcloud'),
+        installer=dict(type='str', required=False, default='tripleo')
     )
 
     result = dict(
@@ -112,6 +120,7 @@ def run_module():
     parser.release = module.params['release']
     parser.job = module.params['job']
     parser.deployment = module.params['deployment']
+    parser.installer = module.params['installer']
 
     tests = cmd.take_action(parser)
     skipped_tests = [test[0] for test in tests[1]]

roles/list_skipped/tasks/main.yaml

@@ -5,4 +5,5 @@
     release: "{{ list_skipped_release | default(omit) }}"
     job: "{{ list_skipped_job | default(omit) }}"
     deployment: "{{ list_skipped_deployment | default(omit) }}"
+    installer: "{{ list_skipped_installer | default(omit) }}"
   register: tempest_skip_register

tempest_skip/list_yaml.py

@@ -42,37 +42,28 @@ class ListSkippedYaml(Lister):
             tests = yaml_file.get('known_failures', [])
 
             if parsed_args.job:
-                self.parsed_job = parsed_args.job
+                tests = [test for test in tests
-                tests = list(filter(self._filter_jobs, tests))
+                         if (not test.get('jobs', []) or (
+                             parsed_args.job in test.get('jobs')))]
 
             if parsed_args.release:
-                new_tests = []
+                tests = [test for test in tests
-                for test in tests:
+                         if [release for release in test.get('releases', [])
-                    for release in test.get('releases', []):
+                             if release['name'] == parsed_args.release]]
-                        if release.get('name') == parsed_args.release:
+
-                            new_tests.append(test)
+            tests = [test for test in tests
-                tests = new_tests
+                     if [release for release in test.get('releases', [])
+                         if parsed_args.installer in
+                         release.get('installers', ['tripleo', 'osp'])]]
 
             if parsed_args.deployment:
-                self.parsed_deployment = parsed_args.deployment
+                tests = [test for test in tests
-                tests = list(filter(self._filter_deployment, tests))
+                         if (not test.get('deployment', []) or (
+                             parsed_args.deployment in
+                             test.get('deployment')))]
             return tests
         return []
 
-    def _filter_deployment(self, test):
-        if not test.get('deployment', []):
-            return True
-        if self.parsed_deployment in test.get('deployment'):
-            return True
-        return False
-
-    def _filter_jobs(self, test):
-        if not test.get('jobs', []):
-            return True
-        if self.parsed_job in test.get('jobs'):
-            return True
-        return False
-
     def get_parser(self, prog_name):
         parser = super(ListSkippedYaml, self).get_parser(prog_name)
         parser.add_argument('--file', dest='file',
@@ -91,4 +82,9 @@ class ListSkippedYaml(Lister):
         parser.add_argument('--deployment', dest='deployment',
                             help='List the tests to be skipped in the '
                                  'given deployment')
+        parser.add_argument('--installer', dest='installer',
+                            default=None, help='Tests to be skipped for a '
+                                                'particular installer. Use '
+                                                'tripleo for upstream, and osp'
+                                                ' for downstream')
         return parser

tempest_skip/tests/test_list_yaml.py

@@ -48,6 +48,31 @@ class TestListSkippedYaml(base.TestCase):
                     lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166'
                 jobs:
                   - 'job1'
+              - test: 'tempest_skip.tests.test_list_yaml_4'
+                deployment:
+                  - 'overcloud'
+                releases:
+                  - name: 'train'
+                    reason: 'Test failing on train release'
+                    lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166'
+                    installers:
+                      - 'osp'
+                jobs:
+                  - 'job3'
+              - test: 'tempest_skip.tests.test_list_yaml_5'
+                deployment:
+                  - 'overcloud'
+                releases:
+                  - name: 'train'
+                    reason: 'Test failing on train release'
+                    lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166'
+                    installers:
+                      - 'osp'
+                      - 'tripleo'
+                  - name: 'wallaby'
+                    reason: 'Test failing on train release'
+                    lp: 'https://bugs.launchpad.net/tripleo/+bug/1832166'
+                jobs: []
         """
         self.path = self.write_yaml_file(self.list_file)
 
@@ -57,13 +82,15 @@ class TestListSkippedYaml(base.TestCase):
         self.parser.release = None
         self.parser.deployment = None
         self.parser.job = None
+        self.parser.installer = 'tripleo'
 
     def test_list_yaml(self):
         cmd_result = self.cmd.take_action(self.parser)
 
         exptected = [('tempest_skip.tests.test_list_yaml',),
                      ('tempest_skip.tests.test_list_yaml_2',),
-                     ('tempest_skip.tests.test_list_yaml_3',)]
+                     ('tempest_skip.tests.test_list_yaml_3',),
+                     ('tempest_skip.tests.test_list_yaml_5',)]
         list_tests = [test for test in cmd_result[1]]
         self.assertEqual(exptected, list_tests)
 
@@ -72,7 +99,8 @@ class TestListSkippedYaml(base.TestCase):
         self.parser.job = None
 
         cmd_result = self.cmd.take_action(self.parser)
-        exptected = [('tempest_skip.tests.test_list_yaml_3',)]
+        exptected = [('tempest_skip.tests.test_list_yaml_3',),
+                     ('tempest_skip.tests.test_list_yaml_5',)]
         list_tests = [test for test in cmd_result[1]]
         self.assertEqual(exptected, list_tests)
 
@@ -102,6 +130,35 @@ class TestListSkippedYaml(base.TestCase):
         list_tests = [test for test in cmd_result[1]]
         self.assertEqual(expected, list_tests)
 
+    def test_list_yaml_with_installer(self):
+        self.parser.installer = 'osp'
+        self.parser.release = 'master'
+        cmd_result = self.cmd.take_action(self.parser)
+        expected = [('tempest_skip.tests.test_list_yaml',),
+                    ('tempest_skip.tests.test_list_yaml_2',)]
+        list_tests = [test for test in cmd_result[1]]
+        self.assertEqual(expected, list_tests)
+
+    def test_list_yaml_with_installer_and_deployment(self):
+        self.parser.installer = 'osp'
+        self.parser.deployment = 'overcloud'
+        cmd_result = self.cmd.take_action(self.parser)
+        expected = [('tempest_skip.tests.test_list_yaml',),
+                    ('tempest_skip.tests.test_list_yaml_2',),
+                    ('tempest_skip.tests.test_list_yaml_3',),
+                    ('tempest_skip.tests.test_list_yaml_4',),
+                    ('tempest_skip.tests.test_list_yaml_5',)]
+        list_tests = [test for test in cmd_result[1]]
+        self.assertEqual(expected, list_tests)
+
+    def test_list_yaml_with_installer_and_invalid_deployment(self):
+        self.parser.installer = 'osp'
+        self.parser.deployment = 'underclouds'
+        cmd_result = self.cmd.take_action(self.parser)
+        expected = []
+        list_tests = [test for test in cmd_result[1]]
+        self.assertEqual(expected, list_tests)
+
     def test_list_yaml_with_release_not_found(self):
         self.parser.release = 'not-found'
         self.parser.job = None
@@ -116,7 +173,8 @@ class TestListSkippedYaml(base.TestCase):
         cmd_result = self.cmd.take_action(self.parser)
         exptected = [('tempest_skip.tests.test_list_yaml',),
                      ('tempest_skip.tests.test_list_yaml_2',),
-                     ('tempest_skip.tests.test_list_yaml_3',)]
+                     ('tempest_skip.tests.test_list_yaml_3',),
+                     ('tempest_skip.tests.test_list_yaml_5',)]
         list_tests = [test for test in cmd_result[1]]
         self.assertEqual(exptected, list_tests)
 
@@ -124,7 +182,8 @@ class TestListSkippedYaml(base.TestCase):
         self.parser.job = 'job2'
         cmd_result = self.cmd.take_action(self.parser)
         exptected = [('tempest_skip.tests.test_list_yaml',),
-                     ('tempest_skip.tests.test_list_yaml_2',)]
+                     ('tempest_skip.tests.test_list_yaml_2',),
+                     ('tempest_skip.tests.test_list_yaml_5',)]
         list_tests = [test for test in cmd_result[1]]
         self.assertEqual(exptected, list_tests)
 
@@ -132,7 +191,8 @@ class TestListSkippedYaml(base.TestCase):
         self.parser.job = 'job1'
         self.parser.release = 'train'
         cmd_result = self.cmd.take_action(self.parser)
-        exptected = [('tempest_skip.tests.test_list_yaml_3',)]
+        exptected = [('tempest_skip.tests.test_list_yaml_3',),
+                     ('tempest_skip.tests.test_list_yaml_5',)]
         list_tests = [test for test in cmd_result[1]]
         self.assertEqual(exptected, list_tests)
 

tempest_skip/tests/test_validate.py

@@ -110,6 +110,9 @@ class TestValidateSkipped(TestValidate):
                   - name: 'master'
                     lp: 'https://launchpad.net/bugs/1'
                     reason: 'Test with launchpad'
+                    installers:
+                      - 'tripleo'
+                      - 'osp'
                   - name: 'train'
                     bz: 'https://bugzilla.redhat.com/1'
                     reason: 'Test with bugzilla'

tempest_skip/validate.py

@@ -38,7 +38,8 @@ class Validate(Command):
                     v.Required('name'): str,
                     v.Required(v.SomeOf(
                         validators=[v.Any('lp', 'bz')], min_valid=1)): v.Url(),
-                    v.Required('reason'): str
+                    v.Required('reason'): str,
+                    v.Optional('installers'): [str]
                 })
             ],
             v.Optional('jobs'): [str]