Allow layered templates

Accept multiple template invocations per project, and also allow
adding individual jobs to a project that uses templates.

Change-Id: I6c668dd434c12bec96b9a27afd9fd2eca7a11d0a

doc/source/zuul.rst

@@ -637,6 +637,30 @@ key::
 You can pass several parameters to a template. A ``parameter`` value will be
 used for expansion of ``{parameter}`` in the template strings.
 
+Multiple templates can be combined in a project, and the jobs from all
+of those templates will be added to the project.  Individual jobs may
+also be added::
+
+  projects:
+   - name: plugin/foobar
+     template:
+      - name: plugin-triggering
+        jobprefix: plugin-foobar
+      - name: plugin-extras
+        jobprefix: plugin-foobar
+     check:
+      - foobar-extra-special-job
+
+The order of the jobs listed in the project (which only affects the
+order of jobs listed on the report) will be the jobs from each
+template in the order listed, followed by any jobs individually listed
+for the project.
+
+Note that if multiple templates are used for a project and one
+template specifies a job that is also specified in another template,
+or specified in the project itself, those jobs will be duplicated in
+the resulting project configuration.
+
 logging.conf
 ~~~~~~~~~~~~
 This file is optional.  If provided, it should be a standard

tests/fixtures/layout.yaml

@@ -110,6 +110,16 @@ project-templates:
     check:
      - '{projectname}-test1'
      - '{projectname}-test2'
+  - name: test-three-and-four
+    check:
+     - '{projectname}-test3'
+     - '{projectname}-test4'
+  - name: test-five
+    check:
+     - '{projectname}-test5'
+  - name: test-five-also
+    check:
+     - '{projectname}-test5'
 
 projects:
   - name: org/project
@@ -195,8 +205,21 @@ projects:
 
   - name: org/templated-project
     template:
-     - name: test-one-and-two
-       projectname: project
+      - name: test-one-and-two
+        projectname: project
+
+  - name: org/layered-project
+    template:
+      - name: test-one-and-two
+        projectname: project
+      - name: test-three-and-four
+        projectname: project
+      - name: test-five
+        projectname: project
+      - name: test-five-also
+        projectname: project
+    check:
+      - project-test6
 
   - name: org/node-project
     gate:

tests/test_scheduler.py

@@ -766,6 +766,7 @@ class TestScheduler(testtools.TestCase):
         self.init_repo("org/one-job-project")
         self.init_repo("org/nonvoting-project")
         self.init_repo("org/templated-project")
+        self.init_repo("org/layered-project")
         self.init_repo("org/node-project")
         self.init_repo("org/conflict-project")
 
@@ -1958,6 +1959,32 @@ class TestScheduler(testtools.TestCase):
         self.assertEqual(self.getJobFromHistory('project-test2').result,
                          'SUCCESS')
 
+    def test_layered_templates(self):
+        "Test whether a job generated via a template can be launched"
+
+        A = self.fake_gerrit.addFakeChange(
+            'org/layered-project', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(self.getJobFromHistory('project-test1').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory('project-test2').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory('project-test3').result,
+                         'SUCCESS')
+        self.assertEqual(self.getJobFromHistory('project-test4').result,
+                         'SUCCESS')
+        # project-test5 should run twice because two templates define it
+        test5_count = 0
+        for job in self.worker.build_history:
+            if job.name == 'project-test5':
+                test5_count += 1
+                self.assertEqual(job.result, 'SUCCESS')
+        self.assertEqual(test5_count, 2)
+        self.assertEqual(self.getJobFromHistory('project-test6').result,
+                         'SUCCESS')
+
     def test_dependent_changes_dequeue(self):
         "Test that dependent patches are not needlessly tested"
 

zuul/scheduler.py

@@ -287,15 +287,30 @@ class Scheduler(threading.Thread):
         for config_project in data.get('projects', []):
             project = Project(config_project['name'])
 
-            for requested_template in config_project.get('template', []):
+            # This is reversed due to the prepend operation below, so
+            # the ultimate order is templates (in order) followed by
+            # statically defined jobs.
+            for requested_template in reversed(
+                config_project.get('template', [])):
                 # Fetch the template from 'project-templates'
                 tpl = project_templates.get(
                     requested_template.get('name'))
                 # Expand it with the project context
                 expanded = deep_format(tpl, requested_template)
-                # Finally merge the expansion with whatever has been already
-                # defined for this project
-                config_project.update(expanded)
+                # Finally merge the expansion with whatever has been
+                # already defined for this project.  Prepend our new
+                # jobs to existing ones (which may have been
+                # statically defined or defined by other templates).
+                for pipeline in layout.pipelines.values():
+                    if pipeline.name in expanded:
+                        config_project.update(
+                            {pipeline.name: expanded[pipeline.name] +
+                             config_project.get(pipeline.name, [])})
+            # TODO: future enhancement -- add an option to the
+            # template block to indicate that duplicate jobs should be
+            # merged (especially to handle the case where they have
+            # children and you want all of the children to run after a
+            # single run of the parent).
 
             layout.projects[config_project['name']] = project
             mode = config_project.get('merge-mode', 'merge-resolve')