diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst index d79600f878..96faf86c5b 100644 --- a/doc/source/zuul.rst +++ b/doc/source/zuul.rst @@ -504,6 +504,40 @@ can help avoid running unnecessary jobs. .. seealso:: The OpenStack Zuul configuration for a comprehensive example: https://github.com/openstack-infra/config/blob/master/modules/openstack_project/files/zuul/layout.yaml +Project Templates +""""""""""""""""" + +Whenever you have lot of similiar projects (such as plugins for a project) you +will most probably want to use the same pipeline configurations. The +project templates let you define pipelines and job name templates to trigger. +One can then just apply the template on its project which make it easier to +update several similiar projects. As an example:: + + project-templates: + # Name of the template + - name: plugin-triggering + # Definition of pipelines just like for a `project` + check: + - '{jobprefix}-merge': + - '{jobprefix}-pep8' + - '{jobprefix}-pyflakes' + gate: + - '{jobprefix}-merge': + - '{jobprefix}-unittest' + - '{jobprefix}-pep8' + - '{jobprefix}-pyflakes' + +In your projects definition, you will then apply the template using the template +key:: + + projects: + - name: plugin/foobar + template: + - name: plugin-triggering + jobprefix: plugin-foobar + +You can pass several parameters to a template. A ``parameter`` value will be +used for expansion of ``{parameter}`` in the template strings. logging.conf ~~~~~~~~~~~~ diff --git a/tests/fixtures/layout.yaml b/tests/fixtures/layout.yaml index 5276d8357c..ecdd2da59f 100644 --- a/tests/fixtures/layout.yaml +++ b/tests/fixtures/layout.yaml @@ -47,6 +47,12 @@ jobs: files: - '.*-requires' +project-templates: + - name: test-one-and-two + check: + - '{projectname}-test1' + - '{projectname}-test2' + projects: - name: org/project merge-mode: cherry-pick @@ -124,3 +130,8 @@ projects: - nonvoting-project-test2 post: - nonvoting-project-post + + - name: org/templated-project + template: + - name: test-one-and-two + projectname: project diff --git a/tests/fixtures/layouts/bad_template1.yaml b/tests/fixtures/layouts/bad_template1.yaml new file mode 100644 index 0000000000..43da793c26 --- /dev/null +++ b/tests/fixtures/layouts/bad_template1.yaml @@ -0,0 +1,19 @@ +# Template is going to be called but missing a parameter + +pipelines: + - name: 'check' + manager: IndependentPipelineManager + trigger: + - event: patchset-created + +project-templates: + - name: template-generic + check: + # Template uses the 'project' parameter' which must + - '{project}-merge' + +projects: + - name: organization/project + template: + - name: template-generic + # Here we 'forgot' to pass 'project' diff --git a/tests/fixtures/layouts/bad_template2.yaml b/tests/fixtures/layouts/bad_template2.yaml new file mode 100644 index 0000000000..0e40d2d943 --- /dev/null +++ b/tests/fixtures/layouts/bad_template2.yaml @@ -0,0 +1,22 @@ +# Template is going to be called with an extra parameter + +pipelines: + - name: 'check' + manager: IndependentPipelineManager + trigger: + - event: patchset-created + +project-templates: + - name: template-generic + check: + # Template only uses the 'project' parameter' + - '{project}-merge' + +projects: + - name: organization/project + template: + - name: template-generic + project: 'MyProjectName' + # Feed an extra parameters which is not going to be used + # by the template. That is an error. + extraparam: 'IShouldNotBeSet' diff --git a/tests/fixtures/layouts/bad_template3.yaml b/tests/fixtures/layouts/bad_template3.yaml new file mode 100644 index 0000000000..70412b86d3 --- /dev/null +++ b/tests/fixtures/layouts/bad_template3.yaml @@ -0,0 +1,13 @@ +# Template refers to an unexisting pipeline + +pipelines: + # We have no pipelines at all + +project-templates: + - name: template-generic + unexisting-pipeline: # pipeline does not exist + +projects: + - name: organization/project + template: + - name: template-generic diff --git a/tests/fixtures/layouts/good_template1.yaml b/tests/fixtures/layouts/good_template1.yaml new file mode 100644 index 0000000000..1d179f7923 --- /dev/null +++ b/tests/fixtures/layouts/good_template1.yaml @@ -0,0 +1,16 @@ +pipelines: + - name: 'check' + manager: IndependentPipelineManager + trigger: + - event: patchset-created + +project-templates: + - name: template-generic + check: + - '{project}-merge' + +projects: + - name: organization/project + template: + - name: template-generic + project: 'myproject' diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 37526f4ac5..26c2710ed8 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -689,6 +689,7 @@ class testScheduler(unittest.TestCase): init_repo("org/project3") init_repo("org/one-job-project") init_repo("org/nonvoting-project") + init_repo("org/templated-project") self.config = CONFIG self.statsd = FakeStatsd() @@ -1376,6 +1377,20 @@ class testScheduler(unittest.TestCase): assert B.reported == 2 self.assertEmptyQueues() + def test_job_from_templates_launched(self): + "Test whether a job generated via a template can be launched" + A = self.fake_gerrit.addFakeChange( + 'org/templated-project', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + jobs = self.fake_jenkins.job_history + job_names = [x.name for x in jobs] + + assert 'project-test1' in job_names + assert 'project-test2' in job_names + assert jobs[0].result == 'SUCCESS' + assert jobs[1].result == 'SUCCESS' + def test_dependent_changes_dequeue(self): "Test that dependent patches are not needlessly tested" A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') diff --git a/zuul/layoutvalidator.py b/zuul/layoutvalidator.py index 963359ebe9..5588afe8f5 100644 --- a/zuul/layoutvalidator.py +++ b/zuul/layoutvalidator.py @@ -1,4 +1,6 @@ # Copyright 2013 OpenStack Foundation +# Copyright 2013 Antoine "hashar" Musso +# Copyright 2013 Wikimedia Foundation Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -13,6 +15,7 @@ # under the License. import voluptuous as v +import string # Several forms accept either a single item or a list, this makes @@ -55,6 +58,9 @@ class LayoutSchema(object): } pipelines = [pipeline] + project_template = {v.Required('name'): str} + project_templates = [project_template] + job = {v.Required('name'): str, 'failure-message': str, 'success-message': str, @@ -80,21 +86,90 @@ class LayoutSchema(object): else: self.job_name.validate(path, self.job_name.schema, value) + def validateTemplateCalls(self, calls): + """ Verify a project pass the parameters required + by a project-template + """ + for call in calls: + schema = self.templates_schemas[call.get('name')] + schema(call) + + def collectFormatParam(self, tree): + """In a nested tree of string, dict and list, find out any named + parameters that might be used by str.format(). This is used to find + out whether projects are passing all the required parameters when + using a project template. + + Returns a set() of all the named parameters found. + """ + parameters = set() + if isinstance(tree, str): + # parse() returns a tuple of + # (literal_text, field_name, format_spec, conversion) + # We are just looking for field_name + parameters = set([t[1] for t in string.Formatter().parse(tree) + if t[1] is not None]) + elif isinstance(tree, list): + for item in tree: + parameters.update(self.collectFormatParam(item)) + elif isinstance(tree, dict): + for item in tree: + parameters.update(self.collectFormatParam(tree[item])) + + return parameters + def getSchema(self, data): pipelines = data.get('pipelines') if not pipelines: pipelines = [] pipelines = [p['name'] for p in pipelines if 'name' in p] + + # Whenever a project uses a template, it better have to exist + project_templates = data.get('project-templates', []) + template_names = [t['name'] for t in project_templates + if 'name' in t] + + # A project using a template must pass all parameters to it. + # We first collect each templates parameters and craft a new + # schema for each of the template. That will later be used + # by validateTemplateCalls(). + self.templates_schemas = {} + for t_name in template_names: + # Find out the parameters used inside each templates: + template = [t for t in project_templates + if t['name'] == t_name] + template_parameters = self.collectFormatParam(template) + + # Craft the templates schemas + schema = {v.Required('name'): v.Any(*template_names)} + for required_param in template_parameters: + # add this template parameters as requirements: + schema.update({v.Required(required_param): str}) + + # Register the schema for validateTemplateCalls() + self.templates_schemas[t_name] = v.Schema(schema) + project = {'name': str, 'merge-mode': v.Any('cherry-pick'), + 'template': self.validateTemplateCalls, } + + # And project should refers to existing pipelines for p in pipelines: project[p] = self.validateJob projects = [project] + # Sub schema to validate a project template has existing + # pipelines and jobs. + project_template = {'name': str} + for p in pipelines: + project_template[p] = self.validateJob + project_templates = [project_template] + # Gather our sub schemas schema = v.Schema({'includes': self.includes, v.Required('pipelines'): self.pipelines, 'jobs': self.jobs, + 'project-templates': project_templates, v.Required('projects'): projects, }) return schema @@ -116,3 +191,6 @@ class LayoutValidator(object): if 'jobs' in data: self.checkDuplicateNames(data['jobs'], ['jobs']) self.checkDuplicateNames(data['projects'], ['projects']) + if 'project-templates' in data: + self.checkDuplicateNames( + data['project-templates'], ['project-templates']) diff --git a/zuul/scheduler.py b/zuul/scheduler.py index 0904f37422..f5d12988c8 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -1,5 +1,7 @@ # Copyright 2012 Hewlett-Packard Development Company, L.P. # Copyright 2013 OpenStack Foundation +# Copyright 2013 Antoine "hashar" Musso +# Copyright 2013 Wikimedia Foundation Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -32,6 +34,28 @@ import merger statsd = extras.try_import('statsd.statsd') +def deep_format(obj, paramdict): + """Apply the paramdict via str.format() to all string objects found within + the supplied obj. Lists and dicts are traversed recursively. + + Borrowed from Jenkins Job Builder project""" + if isinstance(obj, str): + ret = obj.format(**paramdict) + elif isinstance(obj, list): + ret = [] + for item in obj: + ret.append(deep_format(item, paramdict)) + elif isinstance(obj, dict): + ret = {} + for item in obj: + exp_item = item.format(**paramdict) + + ret[exp_item] = deep_format(obj[item], paramdict) + else: + ret = obj + return ret + + class Scheduler(threading.Thread): log = logging.getLogger("zuul.Scheduler") @@ -55,6 +79,7 @@ class Scheduler(threading.Thread): self.pipelines = {} self.jobs = {} self.projects = {} + self.project_templates = {} self.metajobs = {} def stop(self): @@ -126,6 +151,16 @@ class Scheduler(threading.Thread): toList(trigger.get('email_filter'))) manager.event_filters.append(f) + for project_template in data.get('project-templates', []): + # Make sure the template only contains valid pipelines + tpl = dict( + (pipe_name, project_template.get(pipe_name)) + for pipe_name in self.pipelines.keys() + if pipe_name in project_template + ) + self.project_templates[project_template.get('name')] \ + = tpl + for config_job in data.get('jobs', []): job = self.getJob(config_job['name']) # Be careful to only set attributes explicitly present on @@ -177,6 +212,17 @@ class Scheduler(threading.Thread): for config_project in data.get('projects', []): project = Project(config_project['name']) + + for requested_template in config_project.get('template', []): + # Fetch the template from 'project-templates' + tpl = self.project_templates.get( + requested_template.get('name')) + # Expand it with the project context + expanded = deep_format(tpl, requested_template) + # Finally merge the expansion with whatever has been already + # defined for this project + config_project.update(expanded) + self.projects[config_project['name']] = project mode = config_project.get('merge-mode') if mode and mode == 'cherry-pick':