From 3e98c02eba0f1a2f8d3649cbd80ba567ffa37cf2 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Mon, 16 Dec 2013 15:25:38 -0800 Subject: [PATCH] Allow layered templates Accept multiple template invocations per project, and also allow adding individual jobs to a project that uses templates. Change-Id: I6c668dd434c12bec96b9a27afd9fd2eca7a11d0a --- doc/source/zuul.rst | 24 ++++++++++++++++++++++++ tests/fixtures/layout.yaml | 27 +++++++++++++++++++++++++-- tests/test_scheduler.py | 27 +++++++++++++++++++++++++++ zuul/scheduler.py | 23 +++++++++++++++++++---- 4 files changed, 95 insertions(+), 6 deletions(-) diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst index afe4cf6097..5560317658 100644 --- a/doc/source/zuul.rst +++ b/doc/source/zuul.rst @@ -637,6 +637,30 @@ key:: You can pass several parameters to a template. A ``parameter`` value will be used for expansion of ``{parameter}`` in the template strings. +Multiple templates can be combined in a project, and the jobs from all +of those templates will be added to the project. Individual jobs may +also be added:: + + projects: + - name: plugin/foobar + template: + - name: plugin-triggering + jobprefix: plugin-foobar + - name: plugin-extras + jobprefix: plugin-foobar + check: + - foobar-extra-special-job + +The order of the jobs listed in the project (which only affects the +order of jobs listed on the report) will be the jobs from each +template in the order listed, followed by any jobs individually listed +for the project. + +Note that if multiple templates are used for a project and one +template specifies a job that is also specified in another template, +or specified in the project itself, those jobs will be duplicated in +the resulting project configuration. + logging.conf ~~~~~~~~~~~~ This file is optional. If provided, it should be a standard diff --git a/tests/fixtures/layout.yaml b/tests/fixtures/layout.yaml index dc659fb911..2e113c2f0e 100644 --- a/tests/fixtures/layout.yaml +++ b/tests/fixtures/layout.yaml @@ -110,6 +110,16 @@ project-templates: check: - '{projectname}-test1' - '{projectname}-test2' + - name: test-three-and-four + check: + - '{projectname}-test3' + - '{projectname}-test4' + - name: test-five + check: + - '{projectname}-test5' + - name: test-five-also + check: + - '{projectname}-test5' projects: - name: org/project @@ -195,8 +205,21 @@ projects: - name: org/templated-project template: - - name: test-one-and-two - projectname: project + - name: test-one-and-two + projectname: project + + - name: org/layered-project + template: + - name: test-one-and-two + projectname: project + - name: test-three-and-four + projectname: project + - name: test-five + projectname: project + - name: test-five-also + projectname: project + check: + - project-test6 - name: org/node-project gate: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index c15d70c9b7..8fb84195b5 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -766,6 +766,7 @@ class TestScheduler(testtools.TestCase): self.init_repo("org/one-job-project") self.init_repo("org/nonvoting-project") self.init_repo("org/templated-project") + self.init_repo("org/layered-project") self.init_repo("org/node-project") self.init_repo("org/conflict-project") @@ -1958,6 +1959,32 @@ class TestScheduler(testtools.TestCase): self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') + def test_layered_templates(self): + "Test whether a job generated via a template can be launched" + + A = self.fake_gerrit.addFakeChange( + 'org/layered-project', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertEqual(self.getJobFromHistory('project-test1').result, + 'SUCCESS') + self.assertEqual(self.getJobFromHistory('project-test2').result, + 'SUCCESS') + self.assertEqual(self.getJobFromHistory('project-test3').result, + 'SUCCESS') + self.assertEqual(self.getJobFromHistory('project-test4').result, + 'SUCCESS') + # project-test5 should run twice because two templates define it + test5_count = 0 + for job in self.worker.build_history: + if job.name == 'project-test5': + test5_count += 1 + self.assertEqual(job.result, 'SUCCESS') + self.assertEqual(test5_count, 2) + self.assertEqual(self.getJobFromHistory('project-test6').result, + 'SUCCESS') + def test_dependent_changes_dequeue(self): "Test that dependent patches are not needlessly tested" diff --git a/zuul/scheduler.py b/zuul/scheduler.py index f8dd6e1981..6eca846c54 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -287,15 +287,30 @@ class Scheduler(threading.Thread): for config_project in data.get('projects', []): project = Project(config_project['name']) - for requested_template in config_project.get('template', []): + # This is reversed due to the prepend operation below, so + # the ultimate order is templates (in order) followed by + # statically defined jobs. + for requested_template in reversed( + config_project.get('template', [])): # Fetch the template from 'project-templates' tpl = project_templates.get( requested_template.get('name')) # Expand it with the project context expanded = deep_format(tpl, requested_template) - # Finally merge the expansion with whatever has been already - # defined for this project - config_project.update(expanded) + # Finally merge the expansion with whatever has been + # already defined for this project. Prepend our new + # jobs to existing ones (which may have been + # statically defined or defined by other templates). + for pipeline in layout.pipelines.values(): + if pipeline.name in expanded: + config_project.update( + {pipeline.name: expanded[pipeline.name] + + config_project.get(pipeline.name, [])}) + # TODO: future enhancement -- add an option to the + # template block to indicate that duplicate jobs should be + # merged (especially to handle the case where they have + # children and you want all of the children to run after a + # single run of the parent). layout.projects[config_project['name']] = project mode = config_project.get('merge-mode', 'merge-resolve')