Merge "Add intermediate flag for jobs"

doc/source/reference/job_def.rst

@@ -153,6 +153,21 @@ Here is an example of two job definitions:
          limitation does not apply to jobs in a
          :term:`config-project`.
 
+   .. attr:: intermediate
+      :default: false
+
+      An intermediate job must be inherited by an abstract job; it can
+      not be inherited by a final job.  All ``intermediate`` jobs
+      *must* also be ``abstract``; a configuration error will be
+      raised if not.
+
+      For example, you may define a base abstract job `foo` and create
+      two abstract jobs that inherit from `foo` called
+      `foo-production` and `foo-development`.  If it would be an error
+      to accidentally inherit from the base job `foo` instead of
+      choosing one of the two variants, `foo` could be marked as
+      ``intermediate``.
+
    .. attr:: success-message
       :default: SUCCESS
 

releasenotes/notes/intermediate-jobs-101e04e7e1497af9.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Jobs may specify the new ``intermediate`` flag to note they may only
+    be inherited by abstract jobs.  This can be useful if building a job
+    hierarchy where wish to limit where a base job is instantiated.

tests/fixtures/config/intermediate/git/common-config/playbooks/base.yaml

@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []

tests/fixtures/config/intermediate/git/common-config/zuul.yaml

@@ -0,0 +1,50 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+
+- job:
+    name: job-abstract-intermediate
+    abstract: true
+    intermediate: true
+
+- job:
+    name: job-abstract
+    abstract: true
+    parent: job-abstract-intermediate
+
+# an intermediate, with an intermediate parent also
+- job:
+    name: job-another-intermediate
+    parent: job-abstract-intermediate
+    abstract: true
+    intermediate: true
+
+- job:
+    name: job-another-abstract
+    parent: job-another-intermediate
+    abstract: true
+
+- job:
+    name: job-actual
+    parent: job-another-abstract
+    run: playbooks/base.yaml
+
+- project:
+    name: org/project
+    check:
+      jobs: []
+

tests/fixtures/config/intermediate/git/org_project/README

@@ -0,0 +1 @@
+test

tests/fixtures/config/intermediate/main.yaml

@@ -0,0 +1,8 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project

tests/unit/test_v3.py

@@ -221,6 +221,81 @@ class TestAbstract(ZuulTestCase):
         self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '1')
 
 
+class TestIntermediate(ZuulTestCase):
+    tenant_config_file = 'config/intermediate/main.yaml'
+
+    def test_intermediate_fail(self):
+        # you can not instantiate from an intermediate job
+        in_repo_conf = textwrap.dedent(
+            """
+            - job:
+                name: job-instantiate-intermediate
+                parent: job-abstract-intermediate
+
+            - project:
+                check:
+                  jobs:
+                    - job-instantiate-intermediate
+            """)
+
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(A.reported, 1)
+        self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '-1')
+        self.assertIn('may only inherit to another abstract job',
+                      A.messages[0])
+
+    def test_intermediate_config_fail(self):
+        # an intermediate job must also be abstract
+        in_repo_conf = textwrap.dedent(
+            """
+            - job:
+                name: job-intermediate-but-not-abstract
+                intermediate: true
+                abstract: false
+
+            - project:
+                check:
+                  jobs:
+                    - job-intermediate-but-not-abstract
+            """)
+
+        file_dict = {'zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(A.reported, 1)
+        self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '-1')
+        self.assertIn('An intermediate job must also be abstract',
+                      A.messages[0])
+
+    def test_intermediate_several(self):
+        # test passing through several intermediate jobs
+        in_repo_conf = textwrap.dedent(
+            """
+            - project:
+                name: org/project
+                check:
+                  jobs:
+                    - job-actual
+            """)
+
+        file_dict = {'.zuul.yaml': in_repo_conf}
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+                                           files=file_dict)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(A.reported, 1)
+        self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '1')
+
+
 class TestFinal(ZuulTestCase):
 
     tenant_config_file = 'config/final/main.yaml'

tests/unit/test_web.py

@@ -341,6 +341,7 @@ class TestWeb(BaseTestWeb):
                 'dependencies': [],
                 'description': None,
                 'files': [],
+                'intermediate': False,
                 'irrelevant_files': [],
                 'match_on_config_updates': True,
                 'final': False,
@@ -385,6 +386,7 @@ class TestWeb(BaseTestWeb):
                 'dependencies': [],
                 'description': None,
                 'files': [],
+                'intermediate': False,
                 'irrelevant_files': [],
                 'match_on_config_updates': True,
                 'final': False,
@@ -434,6 +436,7 @@ class TestWeb(BaseTestWeb):
                 'description': None,
                 'files': [],
                 'final': False,
+                'intermediate': False,
                 'irrelevant_files': [],
                 'match_on_config_updates': True,
                 'name': 'test-job',
@@ -555,6 +558,7 @@ class TestWeb(BaseTestWeb):
                   'description': None,
                   'files': [],
                   'final': False,
+                  'intermediate': False,
                   'irrelevant_files': [],
                   'match_on_config_updates': True,
                   'name': 'project-merge',
@@ -592,6 +596,7 @@ class TestWeb(BaseTestWeb):
                   'description': None,
                   'files': [],
                   'final': False,
+                  'intermediate': False,
                   'irrelevant_files': [],
                   'match_on_config_updates': True,
                   'name': 'project-test1',
@@ -629,6 +634,7 @@ class TestWeb(BaseTestWeb):
                   'description': None,
                   'files': [],
                   'final': False,
+                  'intermediate': False,
                   'irrelevant_files': [],
                   'match_on_config_updates': True,
                   'name': 'project-test2',
@@ -666,6 +672,7 @@ class TestWeb(BaseTestWeb):
                   'description': None,
                   'files': [],
                   'final': False,
+                  'intermediate': False,
                   'irrelevant_files': [],
                   'match_on_config_updates': True,
                   'name': 'project1-project2-integration',
@@ -723,6 +730,7 @@ class TestWeb(BaseTestWeb):
                              'description': None,
                              'files': [],
                              'final': False,
+                             'intermediate': False,
                              'irrelevant_files': [],
                              'match_on_config_updates': True,
                              'name': 'project-post',

zuul/configloader.py

@@ -603,6 +603,7 @@ class JobParser(object):
                       'final': bool,
                       'abstract': bool,
                       'protected': bool,
+                      'intermediate': bool,
                       'requires': to_list(str),
                       'provides': to_list(str),
                       'failure-message': str,
@@ -655,6 +656,7 @@ class JobParser(object):
         'final',
         'abstract',
         'protected',
+        'intermediate',
         'timeout',
         'post-timeout',
         'workspace',
@@ -810,6 +812,9 @@ class JobParser(object):
                                             job.roles, secrets)
                 job.run = job.run + (run,)
 
+        if conf.get('intermediate', False) and not conf.get('abstract', False):
+            raise Exception("An intermediate job must also be abstract")
+
         for k in self.simple_attributes:
             a = k.replace('-', '_')
             if k in conf:

zuul/model.py

@@ -1205,6 +1205,7 @@ class Job(ConfigObject):
             attempts=3,
             final=False,
             abstract=False,
+            intermediate=False,
             protected=None,
             roles=(),
             required_projects={},
@@ -1268,6 +1269,7 @@ class Job(ConfigObject):
         d['group_variables'] = self.group_variables
         d['final'] = self.final
         d['abstract'] = self.abstract
+        d['intermediate'] = self.intermediate
         d['protected'] = self.protected
         d['voting'] = self.voting
         d['timeout'] = self.timeout
@@ -1559,7 +1561,8 @@ class Job(ConfigObject):
 
         for k in self.execution_attributes:
             if (other._get(k) is not None and
-                k not in set(['final', 'abstract', 'protected'])):
+                k not in set(['final', 'abstract', 'protected',
+                              'intermediate'])):
                 if self.final:
                     raise Exception("Unable to modify final job %s attribute "
                                     "%s=%s with variant %s" % (
@@ -1592,6 +1595,19 @@ class Job(ConfigObject):
         elif other.abstract:
             self.abstract = True
 
+        # An intermediate job may only be inherited by an abstract
+        # job.  Note intermediate jobs must be also be abstract, that
+        # has been enforced during config reading.  Similar to
+        # abstract, it is cleared by inheriting.
+        if self.intermediate and not other.abstract:
+            raise Exception("Intermediate job %s may only inherit "
+                            "to another abstract job" %
+                            (repr(self)))
+        if other.name != self.name:
+            self.intermediate = other.intermediate
+        elif other.intermediate:
+            self.intermediate = True
+
         # Protected may only be set to true
         if other.protected is not None:
             # don't allow to reset protected flag