zuul
/
zuul
Code Issues Proposed changes

Add variables to project 

This adds a "vars:" entry to projects and project-templates to make
available global values for each job.  This can be useful to avoid
repeating the same variable definitions for the same job in different
pipelines, or to pass a project-specific value to jobs in a templating
situation.

Change-Id: I9fb468743a21067773979d113e714b5c3e908d02
This commit is contained in:
Ian Wienand 2018-07-20 15:17:28 +10:00 committed by Monty Taylor
parent 0e48355003
commit 8d80ec2ba8
No known key found for this signature in database
GPG Key ID: 7BAE94BC7141A594
7 changed files with 155 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
doc/source/user/config.rst
View File

@ -1203,6 +1203,13 @@ pipeline.
Cherry-picks each change onto the branch rather than
performing any merges.
.. attr:: vars
:default: None
A dictionary of variables to be made available for all jobs in
all pipelines of this project. For more information see
:ref:`variable inheritance <user_jobs_variable_inheritance>`.
.. attr:: <pipeline>
Each pipeline that the project participates in should have an

43
doc/source/user/jobs.rst
View File

@ -78,6 +78,8 @@ behavior.
.. TODO: document src (and logs?) directory
.. _user_jobs_variable_inheritance:
Variables
---------
@ -89,6 +91,7 @@ order of precedence is:
#. :ref:`Job extra variables <user_jobs_job_extra_variables>`
#. :ref:`Secrets <user_jobs_secrets>`
#. :ref:`Job variables <user_jobs_job_variables>`
#. :ref:`Project variables <user_jobs_project_variables>`
#. :ref:`Parent job results <user_jobs_parent_results>`
Meaning that a site-wide variable with the same name as any other will
@ -156,6 +159,46 @@ They are added to the ``vars`` section of the inventory file under the
``all`` hosts group, so they are available to all hosts. Simply refer
to them by the name specified in the job's ``vars`` section.
.. _user_jobs_project_variables:
Project Variables
~~~~~~~~~~~~~~~~~
Any variables specified in the project definition (using the
:attr:`project.vars` attribute) are available to jobs as Ansible host
variables in the same way as :ref:`job variables
<user_jobs_job_variables>`. Variables set in a ``project-template``
are merged into the project variables when the template is included by
a project.
.. code-block:: yaml
- project-template:
name: sample-template
description: Description
vars:
var_from_template: foo
post:
jobs:
- template_job
release:
jobs:
- template_job
- project:
name: Sample project
description: Description
templates:
- sample-template
vars:
var_for_all_jobs: value
check:
jobs:
- job1
- job2:
vars:
var_for_all_jobs: override
.. _user_jobs_parent_results:
Parent Job Results

5
releasenotes/notes/project-vars-0d57992a7192a62d.yaml Normal file
View File

@ -0,0 +1,5 @@
---
features:
- Project and project-templates may now create variables via a
``vars`` configuration entry. Jobs can access these at runtime
in the same manner as job variables.

40
tests/fixtures/layouts/job-vars.yaml vendored
View File

@ -65,8 +65,45 @@
parent: parentjob
run: playbooks/child3.yaml
- job:
name: override_project_var
parent: parentjob
run: playbooks/override_project_var.yaml
- job:
name: job_from_template1
parent: parentjob
run: playbooks/job_from_template.yaml
- job:
name: job_from_template2
parent: parentjob
run: playbooks/job_from_template.yaml
- project-template:
name: template_with_vars1
vars:
template_var1: 'set_in_template1'
check:
jobs:
- job_from_template1
- project-template:
name: template_with_vars2
vars:
template_var2: 'set_in_template2'
check:
jobs:
- job_from_template2
- project:
name: org/project
description: test description
templates:
- template_with_vars1
- template_with_vars2
vars:
project_var: 'set_in_project'
check:
jobs:
- parentjob
@ -81,6 +118,9 @@
child3: 3
override: 3
child3: 3
- override_project_var:
vars:
project_var: 'override_in_job'
- project:
name: org/project0

14
tests/unit/test_scheduler.py
View File

@ -2643,9 +2643,17 @@ class TestScheduler(ZuulTestCase):
dict(name='child1', result='SUCCESS'),
dict(name='child2', result='SUCCESS'),
dict(name='child3', result='SUCCESS'),
dict(name='override_project_var', result='SUCCESS'),
dict(name='job_from_template1', result='SUCCESS'),
dict(name='job_from_template2', result='SUCCESS'),
], ordered=False)
j = self.getJobFromHistory('parentjob')
rp = set([p['name'] for p in j.parameters['projects']])
self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project')
self.assertEqual(j.parameters['vars']['template_var1'],
'set_in_template1')
self.assertEqual(j.parameters['vars']['template_var2'],
'set_in_template2')
self.assertEqual(j.parameters['vars']['override'], 0)
self.assertEqual(j.parameters['vars']['child1override'], 0)
self.assertEqual(j.parameters['vars']['parent'], 0)
@ -2657,6 +2665,7 @@ class TestScheduler(ZuulTestCase):
'org/project0']))
j = self.getJobFromHistory('child1')
rp = set([p['name'] for p in j.parameters['projects']])
self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project')
self.assertEqual(j.parameters['vars']['override'], 1)
self.assertEqual(j.parameters['vars']['child1override'], 1)
self.assertEqual(j.parameters['vars']['parent'], 0)
@ -2667,6 +2676,7 @@ class TestScheduler(ZuulTestCase):
self.assertEqual(rp, set(['org/project', 'org/project0',
'org/project1']))
j = self.getJobFromHistory('child2')
self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project')
rp = set([p['name'] for p in j.parameters['projects']])
self.assertEqual(j.parameters['vars']['override'], 2)
self.assertEqual(j.parameters['vars']['child1override'], 0)
@ -2678,6 +2688,7 @@ class TestScheduler(ZuulTestCase):
self.assertEqual(rp, set(['org/project', 'org/project0',
'org/project2']))
j = self.getJobFromHistory('child3')
self.assertEqual(j.parameters['vars']['project_var'], 'set_in_project')
rp = set([p['name'] for p in j.parameters['projects']])
self.assertEqual(j.parameters['vars']['override'], 3)
self.assertEqual(j.parameters['vars']['child1override'], 0)
@ -2688,6 +2699,9 @@ class TestScheduler(ZuulTestCase):
self.assertEqual(j.parameters['vars']['child3'], 3)
self.assertEqual(rp, set(['org/project', 'org/project0',
'org/project3']))
j = self.getJobFromHistory('override_project_var')
self.assertEqual(j.parameters['vars']['project_var'],
'override_in_job')
@simple_layout('layouts/job-variants.yaml')
def test_job_branch_variants(self):

19
zuul/configloader.py
View File

@ -806,7 +806,7 @@ class ProjectTemplateParser(object):
self.pcontext = pcontext
self.schema = self.getSchema()
self.not_pipelines = ['name', 'description', 'templates',
'merge-mode', 'default-branch',
'merge-mode', 'default-branch', 'vars',
'_source_context', '_start_mark']
def getSchema(self):
@ -822,6 +822,7 @@ class ProjectTemplateParser(object):
project = {
'name': str,
'description': str,
'vars': dict,
str: pipeline_contents,
'_source_context': model.SourceContext,
'_start_mark': ZuulMark,
@ -854,6 +855,13 @@ class ProjectTemplateParser(object):
if branches:
project_template.setImpliedBranchMatchers(branches)
variables = conf.get('vars', {})
if variables:
if 'zuul' in variables or 'nodepool' in variables:
raise Exception("Variables named 'zuul' or 'nodepool' "
"are not allowed.")
project_template.variables = variables
if freeze:
project_template.freeze()
return project_template
@ -895,6 +903,7 @@ class ProjectParser(object):
project = {
'name': str,
'description': str,
'vars': dict,
'templates': [str],
'merge-mode': vs.Any('merge', 'merge-resolve',
'cherry-pick'),
@ -964,6 +973,13 @@ class ProjectParser(object):
default_branch = conf.get('default-branch', 'master')
project_config.default_branch = default_branch
variables = conf.get('vars', {})
if variables:
if 'zuul' in variables or 'nodepool' in variables:
raise Exception("Variables named 'zuul' or 'nodepool' "
"are not allowed.")
project_config.variables = variables
project_config.freeze()
return project_config
@ -1723,7 +1739,6 @@ class TenantParser(object):
def _addLayoutItems(self, layout, tenant, parsed_config,
skip_pipelines=False, skip_semaphores=False):
# TODO(jeblair): make sure everything needing
# reference_exceptions has it; add tests if needed.
if not skip_pipelines:

31
zuul/model.py
View File

@ -1235,6 +1235,11 @@ class Job(ConfigObject):
self.parent_data = v
self.variables = Job._deepUpdate(self.parent_data, self.variables)
def updateProjectVariables(self, project_vars):
# Merge project/template variables directly into the job
# variables. Job variables override project variables.
self.variables = Job._deepUpdate(project_vars, self.variables)
def updateProjects(self, other_projects):
required_projects = self.required_projects.copy()
required_projects.update(other_projects)
@ -2672,6 +2677,7 @@ class ProjectPipelineConfig(ConfigObject):
self.queue_name = None
self.debug = False
self.debug_messages = []
self.variables = {}
def addDebug(self, msg):
self.debug_messages.append(msg)
@ -2685,6 +2691,12 @@ class ProjectPipelineConfig(ConfigObject):
self.debug = other.debug
self.job_list.inheritFrom(other.job_list)
def updateVariables(self, other):
# We need to keep this separate to update() because we wish to
# apply the project variables all the time, even if its jobs
# only come from templates.
self.variables = Job._deepUpdate(self.variables, other)
class ProjectConfig(ConfigObject):
# Represents a project configuration
@ -2695,6 +2707,7 @@ class ProjectConfig(ConfigObject):
# Pipeline name -> ProjectPipelineConfig
self.pipelines = {}
self.branch_matcher = None
self.variables = {}
# These represent the values from the config file, but should
# not be used directly; instead, use the ProjectMetadata to
# find the computed value from across all project config
@ -2713,6 +2726,7 @@ class ProjectConfig(ConfigObject):
r.templates = self.templates
r.pipelines = self.pipelines
r.branch_matcher = self.branch_matcher
r.variables = self.variables
r.merge_mode = self.merge_mode
r.default_branch = self.default_branch
return r
@ -3218,6 +3232,13 @@ class Layout(object):
self.log.debug("%s item %s" % (msg, item))
project_in_pipeline = True
ppc.update(template_ppc)
ppc.updateVariables(template.variables)
# Now merge in project variables (they will override
# template variables; later job variables may override
# these again)
ppc.updateVariables(pc.variables)
project_ppc = pc.pipelines.get(item.pipeline.name)
if project_ppc:
project_in_pipeline = True
@ -3320,7 +3341,8 @@ class Layout(object):
raise NoMatchingParentError()
return jobs
def _createJobGraph(self, item, job_list, job_graph):
def _createJobGraph(self, item, ppc, job_graph):
job_list = ppc.job_list
change = item.change
pipeline = item.pipeline
item.debug("Freezing job graph")
@ -3401,6 +3423,11 @@ class Layout(object):
if not frozen_job.run:
raise Exception("Job %s does not specify a run playbook" % (
frozen_job.name,))
# Now merge variables set from this parent ppc
# (i.e. project+templates) directly into the job vars
frozen_job.updateProjectVariables(ppc.variables)
job_graph.addJob(frozen_job)
def createJobGraph(self, item, ppc):
@ -3408,7 +3435,7 @@ class Layout(object):
# configured pipeline, if so return an empty JobGraph.
ret = JobGraph()
if ppc:
self._createJobGraph(item, ppc.job_list, ret)
self._createJobGraph(item, ppc, ret)
return ret