diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst index a2baa7a632..c2cca6cdb0 100644 --- a/doc/source/user/config.rst +++ b/doc/source/user/config.rst @@ -1203,6 +1203,13 @@ pipeline. Cherry-picks each change onto the branch rather than performing any merges. + .. attr:: vars + :default: None + + A dictionary of variables to be made available for all jobs in + all pipelines of this project. For more information see + :ref:`variable inheritance `. + .. attr:: Each pipeline that the project participates in should have an diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst index 52f8d0b491..926f501225 100644 --- a/doc/source/user/jobs.rst +++ b/doc/source/user/jobs.rst @@ -78,6 +78,8 @@ behavior. .. TODO: document src (and logs?) directory +.. _user_jobs_variable_inheritance: + Variables --------- @@ -89,6 +91,7 @@ order of precedence is: #. :ref:`Job extra variables ` #. :ref:`Secrets ` #. :ref:`Job variables ` +#. :ref:`Project variables ` #. :ref:`Parent job results ` Meaning that a site-wide variable with the same name as any other will @@ -156,6 +159,46 @@ They are added to the ``vars`` section of the inventory file under the ``all`` hosts group, so they are available to all hosts. Simply refer to them by the name specified in the job's ``vars`` section. +.. _user_jobs_project_variables: + +Project Variables +~~~~~~~~~~~~~~~~~ + +Any variables specified in the project definition (using the +:attr:`project.vars` attribute) are available to jobs as Ansible host +variables in the same way as :ref:`job variables +`. Variables set in a ``project-template`` +are merged into the project variables when the template is included by +a project. + +.. code-block:: yaml + + - project-template: + name: sample-template + description: Description + vars: + var_from_template: foo + post: + jobs: + - template_job + release: + jobs: + - template_job + + - project: + name: Sample project + description: Description + templates: + - sample-template + vars: + var_for_all_jobs: value + check: + jobs: + - job1 + - job2: + vars: + var_for_all_jobs: override + .. _user_jobs_parent_results: Parent Job Results diff --git a/releasenotes/notes/project-vars-0d57992a7192a62d.yaml b/releasenotes/notes/project-vars-0d57992a7192a62d.yaml new file mode 100644 index 0000000000..cf7233703e --- /dev/null +++ b/releasenotes/notes/project-vars-0d57992a7192a62d.yaml @@ -0,0 +1,5 @@ +--- +features: + - Project and project-templates may now create variables via a + ``vars`` configuration entry. Jobs can access these at runtime + in the same manner as job variables. diff --git a/tests/fixtures/layouts/job-vars.yaml b/tests/fixtures/layouts/job-vars.yaml index a5d63b7301..5b12b4969c 100644 --- a/tests/fixtures/layouts/job-vars.yaml +++ b/tests/fixtures/layouts/job-vars.yaml @@ -65,8 +65,45 @@ parent: parentjob run: playbooks/child3.yaml +- job: + name: override_project_var + parent: parentjob + run: playbooks/override_project_var.yaml + +- job: + name: job_from_template1 + parent: parentjob + run: playbooks/job_from_template.yaml + +- job: + name: job_from_template2 + parent: parentjob + run: playbooks/job_from_template.yaml + +- project-template: + name: template_with_vars1 + vars: + template_var1: 'set_in_template1' + check: + jobs: + - job_from_template1 + +- project-template: + name: template_with_vars2 + vars: + template_var2: 'set_in_template2' + check: + jobs: + - job_from_template2 + - project: name: org/project + description: test description + templates: + - template_with_vars1 + - template_with_vars2 + vars: + project_var: 'set_in_project' check: jobs: - parentjob @@ -81,6 +118,9 @@ child3: 3 override: 3 child3: 3 + - override_project_var: + vars: + project_var: 'override_in_job' - project: name: org/project0 diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index f4da1e66a0..6797718397 100644 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -2643,9 +2643,17 @@ class TestScheduler(ZuulTestCase): dict(name='child1', result='SUCCESS'), dict(name='child2', result='SUCCESS'), dict(name='child3', result='SUCCESS'), + dict(name='override_project_var', result='SUCCESS'), + dict(name='job_from_template1', result='SUCCESS'), + dict(name='job_from_template2', result='SUCCESS'), ], ordered=False) j = self.getJobFromHistory('parentjob') rp = set([p['name'] for p in j.parameters['projects']]) + self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project') + self.assertEqual(j.parameters['vars']['template_var1'], + 'set_in_template1') + self.assertEqual(j.parameters['vars']['template_var2'], + 'set_in_template2') self.assertEqual(j.parameters['vars']['override'], 0) self.assertEqual(j.parameters['vars']['child1override'], 0) self.assertEqual(j.parameters['vars']['parent'], 0) @@ -2657,6 +2665,7 @@ class TestScheduler(ZuulTestCase): 'org/project0'])) j = self.getJobFromHistory('child1') rp = set([p['name'] for p in j.parameters['projects']]) + self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project') self.assertEqual(j.parameters['vars']['override'], 1) self.assertEqual(j.parameters['vars']['child1override'], 1) self.assertEqual(j.parameters['vars']['parent'], 0) @@ -2667,6 +2676,7 @@ class TestScheduler(ZuulTestCase): self.assertEqual(rp, set(['org/project', 'org/project0', 'org/project1'])) j = self.getJobFromHistory('child2') + self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project') rp = set([p['name'] for p in j.parameters['projects']]) self.assertEqual(j.parameters['vars']['override'], 2) self.assertEqual(j.parameters['vars']['child1override'], 0) @@ -2678,6 +2688,7 @@ class TestScheduler(ZuulTestCase): self.assertEqual(rp, set(['org/project', 'org/project0', 'org/project2'])) j = self.getJobFromHistory('child3') + self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project') rp = set([p['name'] for p in j.parameters['projects']]) self.assertEqual(j.parameters['vars']['override'], 3) self.assertEqual(j.parameters['vars']['child1override'], 0) @@ -2688,6 +2699,9 @@ class TestScheduler(ZuulTestCase): self.assertEqual(j.parameters['vars']['child3'], 3) self.assertEqual(rp, set(['org/project', 'org/project0', 'org/project3'])) + j = self.getJobFromHistory('override_project_var') + self.assertEqual(j.parameters['vars']['project_var'], + 'override_in_job') @simple_layout('layouts/job-variants.yaml') def test_job_branch_variants(self): diff --git a/zuul/configloader.py b/zuul/configloader.py index cce9d996f0..4a50e36db8 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -806,7 +806,7 @@ class ProjectTemplateParser(object): self.pcontext = pcontext self.schema = self.getSchema() self.not_pipelines = ['name', 'description', 'templates', - 'merge-mode', 'default-branch', + 'merge-mode', 'default-branch', 'vars', '_source_context', '_start_mark'] def getSchema(self): @@ -822,6 +822,7 @@ class ProjectTemplateParser(object): project = { 'name': str, 'description': str, + 'vars': dict, str: pipeline_contents, '_source_context': model.SourceContext, '_start_mark': ZuulMark, @@ -854,6 +855,13 @@ class ProjectTemplateParser(object): if branches: project_template.setImpliedBranchMatchers(branches) + variables = conf.get('vars', {}) + if variables: + if 'zuul' in variables or 'nodepool' in variables: + raise Exception("Variables named 'zuul' or 'nodepool' " + "are not allowed.") + project_template.variables = variables + if freeze: project_template.freeze() return project_template @@ -895,6 +903,7 @@ class ProjectParser(object): project = { 'name': str, 'description': str, + 'vars': dict, 'templates': [str], 'merge-mode': vs.Any('merge', 'merge-resolve', 'cherry-pick'), @@ -964,6 +973,13 @@ class ProjectParser(object): default_branch = conf.get('default-branch', 'master') project_config.default_branch = default_branch + variables = conf.get('vars', {}) + if variables: + if 'zuul' in variables or 'nodepool' in variables: + raise Exception("Variables named 'zuul' or 'nodepool' " + "are not allowed.") + project_config.variables = variables + project_config.freeze() return project_config @@ -1723,7 +1739,6 @@ class TenantParser(object): def _addLayoutItems(self, layout, tenant, parsed_config, skip_pipelines=False, skip_semaphores=False): - # TODO(jeblair): make sure everything needing # reference_exceptions has it; add tests if needed. if not skip_pipelines: diff --git a/zuul/model.py b/zuul/model.py index ae45857516..04a8c9ae1c 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -1235,6 +1235,11 @@ class Job(ConfigObject): self.parent_data = v self.variables = Job._deepUpdate(self.parent_data, self.variables) + def updateProjectVariables(self, project_vars): + # Merge project/template variables directly into the job + # variables. Job variables override project variables. + self.variables = Job._deepUpdate(project_vars, self.variables) + def updateProjects(self, other_projects): required_projects = self.required_projects.copy() required_projects.update(other_projects) @@ -2672,6 +2677,7 @@ class ProjectPipelineConfig(ConfigObject): self.queue_name = None self.debug = False self.debug_messages = [] + self.variables = {} def addDebug(self, msg): self.debug_messages.append(msg) @@ -2685,6 +2691,12 @@ class ProjectPipelineConfig(ConfigObject): self.debug = other.debug self.job_list.inheritFrom(other.job_list) + def updateVariables(self, other): + # We need to keep this separate to update() because we wish to + # apply the project variables all the time, even if its jobs + # only come from templates. + self.variables = Job._deepUpdate(self.variables, other) + class ProjectConfig(ConfigObject): # Represents a project configuration @@ -2695,6 +2707,7 @@ class ProjectConfig(ConfigObject): # Pipeline name -> ProjectPipelineConfig self.pipelines = {} self.branch_matcher = None + self.variables = {} # These represent the values from the config file, but should # not be used directly; instead, use the ProjectMetadata to # find the computed value from across all project config @@ -2713,6 +2726,7 @@ class ProjectConfig(ConfigObject): r.templates = self.templates r.pipelines = self.pipelines r.branch_matcher = self.branch_matcher + r.variables = self.variables r.merge_mode = self.merge_mode r.default_branch = self.default_branch return r @@ -3218,6 +3232,13 @@ class Layout(object): self.log.debug("%s item %s" % (msg, item)) project_in_pipeline = True ppc.update(template_ppc) + ppc.updateVariables(template.variables) + + # Now merge in project variables (they will override + # template variables; later job variables may override + # these again) + ppc.updateVariables(pc.variables) + project_ppc = pc.pipelines.get(item.pipeline.name) if project_ppc: project_in_pipeline = True @@ -3320,7 +3341,8 @@ class Layout(object): raise NoMatchingParentError() return jobs - def _createJobGraph(self, item, job_list, job_graph): + def _createJobGraph(self, item, ppc, job_graph): + job_list = ppc.job_list change = item.change pipeline = item.pipeline item.debug("Freezing job graph") @@ -3401,6 +3423,11 @@ class Layout(object): if not frozen_job.run: raise Exception("Job %s does not specify a run playbook" % ( frozen_job.name,)) + + # Now merge variables set from this parent ppc + # (i.e. project+templates) directly into the job vars + frozen_job.updateProjectVariables(ppc.variables) + job_graph.addJob(frozen_job) def createJobGraph(self, item, ppc): @@ -3408,7 +3435,7 @@ class Layout(object): # configured pipeline, if so return an empty JobGraph. ret = JobGraph() if ppc: - self._createJobGraph(item, ppc.job_list, ret) + self._createJobGraph(item, ppc, ret) return ret