Fix bug in template job yaml expansion.
This bug occurs if one attempts to set a list object on a job template for a parameter that does not occur in the job name. If the parameter is not in the template name then there is no reason to count it among the dimensions used to fanout that template into multiple instances and in fact this would actually lead to confusing duplicate job error messages. Change-Id: I80f026d3bbfbac96c6fc7c01c1c916cf85e1bf10
This commit is contained in:
Wayne
parent
80c6ecf053
commit
c1b290ff13
|
|
@ -21,7 +21,6 @@ import operator
|
|||
import sys
|
||||
import hashlib
|
||||
import yaml
|
||||
import json
|
||||
import xml.etree.ElementTree as XML
|
||||
import xml
|
||||
from xml.dom import minidom
|
||||
|
|
@ -33,7 +32,6 @@ import logging
|
|||
import copy
|
||||
import itertools
|
||||
import fnmatch
|
||||
import six
|
||||
from jenkins_jobs.errors import JenkinsJobsException
|
||||
import jenkins_jobs.local_yaml as local_yaml
|
||||
|
||||
|
|
@ -238,7 +236,7 @@ class YamlParser(object):
|
|||
job["description"] = description + \
|
||||
self.get_managed_string().lstrip()
|
||||
|
||||
def expandYaml(self, jobs_filter=None):
|
||||
def expandYaml(self, jobs_glob=None):
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
|
|
@ -248,7 +246,7 @@ class YamlParser(object):
|
|||
changed = True
|
||||
|
||||
for job in self.data.get('job', {}).values():
|
||||
if jobs_filter and not matches(job['name'], jobs_filter):
|
||||
if jobs_glob and not matches(job['name'], jobs_glob):
|
||||
logger.debug("Ignoring job {0}".format(job['name']))
|
||||
continue
|
||||
logger.debug("Expanding job '{0}'".format(job['name']))
|
||||
|
|
@ -309,7 +307,7 @@ class YamlParser(object):
|
|||
d['name'] = project['name']
|
||||
if template:
|
||||
self.expandYamlForTemplateJob(d, template,
|
||||
jobs_filter)
|
||||
jobs_glob)
|
||||
continue
|
||||
# see if it's a template
|
||||
template = self.getJobTemplate(jobname)
|
||||
|
|
@ -317,7 +315,7 @@ class YamlParser(object):
|
|||
d = {}
|
||||
d.update(project)
|
||||
d.update(jobparams)
|
||||
self.expandYamlForTemplateJob(d, template, jobs_filter)
|
||||
self.expandYamlForTemplateJob(d, template, jobs_glob)
|
||||
else:
|
||||
raise JenkinsJobsException("Failed to find suitable "
|
||||
"template named '{0}'"
|
||||
|
|
@ -332,19 +330,25 @@ class YamlParser(object):
|
|||
self.jobs.remove(job)
|
||||
seen.add(job['name'])
|
||||
|
||||
def expandYamlForTemplateJob(self, project, template, jobs_filter=None):
|
||||
def expandYamlForTemplateJob(self, project, template, jobs_glob=None):
|
||||
dimensions = []
|
||||
template_name = template['name']
|
||||
# reject keys that are not useful during yaml expansion
|
||||
for k in ['jobs']:
|
||||
project.pop(k)
|
||||
for (k, v) in project.items():
|
||||
tmpk = '{{{0}}}'.format(k)
|
||||
if tmpk not in template_name:
|
||||
logger.debug("Variable %s not in name %s, rejecting from job"
|
||||
" matrix expansion.", tmpk, template_name)
|
||||
continue
|
||||
if type(v) == list:
|
||||
dimensions.append(zip([k] * len(v), v))
|
||||
# XXX somewhat hackish to ensure we actually have a single
|
||||
# pass through the loop
|
||||
if len(dimensions) == 0:
|
||||
dimensions = [(("", ""),)]
|
||||
checksums = set([])
|
||||
|
||||
for values in itertools.product(*dimensions):
|
||||
params = copy.deepcopy(project)
|
||||
params = self.applyDefaults(params, template)
|
||||
|
|
@ -362,32 +366,12 @@ class YamlParser(object):
|
|||
params = deep_format(params, params)
|
||||
expanded = deep_format(template, params)
|
||||
|
||||
# Keep track of the resulting expansions to avoid
|
||||
# regenerating the exact same job. Whenever a project has
|
||||
# different values for a parameter and that parameter is not
|
||||
# used in the template, we ended up regenerating the exact
|
||||
# same job.
|
||||
# To achieve that we serialize the expanded template making
|
||||
# sure the dict keys are always in the same order. Then we
|
||||
# record the checksum in an unordered unique set which let
|
||||
# us guarantee a group of parameters will not be added a
|
||||
# second time.
|
||||
uniq = json.dumps(expanded, sort_keys=True)
|
||||
if six.PY3:
|
||||
uniq = uniq.encode('utf-8')
|
||||
checksum = hashlib.md5(uniq).hexdigest()
|
||||
job_name = expanded.get('name')
|
||||
if jobs_glob and not matches(job_name, jobs_glob):
|
||||
continue
|
||||
|
||||
# Lookup the checksum
|
||||
if checksum not in checksums:
|
||||
# We also want to skip expansion whenever the user did
|
||||
# not ask for that job.
|
||||
job_name = expanded.get('name')
|
||||
if jobs_filter and not matches(job_name, jobs_filter):
|
||||
continue
|
||||
|
||||
self.formatDescription(expanded)
|
||||
self.jobs.append(expanded)
|
||||
checksums.add(checksum)
|
||||
self.formatDescription(expanded)
|
||||
self.jobs.append(expanded)
|
||||
|
||||
def get_managed_string(self):
|
||||
# The \n\n is not hard coded, because they get stripped if the
|
||||
|
|
@ -785,15 +769,15 @@ class Builder(object):
|
|||
logger.debug("Ignoring unmanaged jenkins job %s",
|
||||
job['name'])
|
||||
|
||||
def delete_job(self, glob_name, fn=None):
|
||||
def delete_job(self, jobs_glob, fn=None):
|
||||
if fn:
|
||||
self.load_files(fn)
|
||||
self.parser.expandYaml(glob_name)
|
||||
self.parser.expandYaml(jobs_glob)
|
||||
jobs = [j['name']
|
||||
for j in self.parser.jobs
|
||||
if matches(j['name'], [glob_name])]
|
||||
if matches(j['name'], [jobs_glob])]
|
||||
else:
|
||||
jobs = [glob_name]
|
||||
jobs = [jobs_glob]
|
||||
|
||||
if jobs is not None:
|
||||
logger.info("Removing jenkins job(s): %s" % ", ".join(jobs))
|
||||
|
|
@ -807,15 +791,15 @@ class Builder(object):
|
|||
for job in jobs:
|
||||
self.delete_job(job['name'])
|
||||
|
||||
def update_job(self, input_fn, names=None, output=None):
|
||||
def update_job(self, input_fn, jobs_glob=None, output=None):
|
||||
self.load_files(input_fn)
|
||||
self.parser.expandYaml(names)
|
||||
self.parser.expandYaml(jobs_glob)
|
||||
self.parser.generateXML()
|
||||
|
||||
self.parser.xml_jobs.sort(key=operator.attrgetter('name'))
|
||||
|
||||
for job in self.parser.xml_jobs:
|
||||
if names and not matches(job.name, names):
|
||||
if jobs_glob and not matches(job.name, jobs_glob):
|
||||
continue
|
||||
if output:
|
||||
if hasattr(output, 'write'):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,86 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<matrix-project>
|
||||
<executionStrategy class="hudson.matrix.DefaultMatrixExecutionStrategyImpl">
|
||||
<runSequentially>false</runSequentially>
|
||||
</executionStrategy>
|
||||
<combinationFilter/>
|
||||
<axes>
|
||||
<hudson.matrix.TextAxis>
|
||||
<name>PLATFORM</name>
|
||||
<values>
|
||||
<string>centos6-64mdca</string>
|
||||
<string>centos7-64mdca</string>
|
||||
</values>
|
||||
</hudson.matrix.TextAxis>
|
||||
</axes>
|
||||
<actions/>
|
||||
<description><!-- Managed by Jenkins Job Builder --></description>
|
||||
<keepDependencies>false</keepDependencies>
|
||||
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
|
||||
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
|
||||
<concurrentBuild>false</concurrentBuild>
|
||||
<canRoam>true</canRoam>
|
||||
<properties/>
|
||||
<scm class="hudson.scm.NullSCM"/>
|
||||
<builders/>
|
||||
<publishers/>
|
||||
<buildWrappers/>
|
||||
</matrix-project>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<matrix-project>
|
||||
<executionStrategy class="hudson.matrix.DefaultMatrixExecutionStrategyImpl">
|
||||
<runSequentially>false</runSequentially>
|
||||
</executionStrategy>
|
||||
<combinationFilter/>
|
||||
<axes>
|
||||
<hudson.matrix.TextAxis>
|
||||
<name>PLATFORM</name>
|
||||
<values>
|
||||
<string>centos6-64mdca</string>
|
||||
<string>centos7-64mdca</string>
|
||||
</values>
|
||||
</hudson.matrix.TextAxis>
|
||||
</axes>
|
||||
<actions/>
|
||||
<description><!-- Managed by Jenkins Job Builder --></description>
|
||||
<keepDependencies>false</keepDependencies>
|
||||
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
|
||||
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
|
||||
<concurrentBuild>false</concurrentBuild>
|
||||
<canRoam>true</canRoam>
|
||||
<properties/>
|
||||
<scm class="hudson.scm.NullSCM"/>
|
||||
<builders/>
|
||||
<publishers/>
|
||||
<buildWrappers/>
|
||||
</matrix-project>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<matrix-project>
|
||||
<executionStrategy class="hudson.matrix.DefaultMatrixExecutionStrategyImpl">
|
||||
<runSequentially>false</runSequentially>
|
||||
</executionStrategy>
|
||||
<combinationFilter/>
|
||||
<axes>
|
||||
<hudson.matrix.TextAxis>
|
||||
<name>PLATFORM</name>
|
||||
<values>
|
||||
<string>centos4-64mdca</string>
|
||||
<string>centos5-64mdca</string>
|
||||
<string>centos6-64mdca</string>
|
||||
<string>centos7-64mdca</string>
|
||||
</values>
|
||||
</hudson.matrix.TextAxis>
|
||||
</axes>
|
||||
<actions/>
|
||||
<description><!-- Managed by Jenkins Job Builder --></description>
|
||||
<keepDependencies>false</keepDependencies>
|
||||
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
|
||||
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
|
||||
<concurrentBuild>false</concurrentBuild>
|
||||
<canRoam>true</canRoam>
|
||||
<properties/>
|
||||
<scm class="hudson.scm.NullSCM"/>
|
||||
<builders/>
|
||||
<publishers/>
|
||||
<buildWrappers/>
|
||||
</matrix-project>
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
- job-template:
|
||||
name: 'thingerator_{pipeline-type}'
|
||||
project-type: matrix
|
||||
axes:
|
||||
- axis:
|
||||
type: 'user-defined'
|
||||
name: 'PLATFORM'
|
||||
values: '{obj:platform_axis}'
|
||||
|
||||
- project:
|
||||
name: derp
|
||||
jobs:
|
||||
- 'thingerator_{pipeline-type}':
|
||||
pipeline-type: 'foo'
|
||||
platform_axis:
|
||||
- 'centos4-64mdca'
|
||||
- 'centos5-64mdca'
|
||||
- 'centos6-64mdca'
|
||||
- 'centos7-64mdca'
|
||||
|
||||
- 'thingerator_{pipeline-type}':
|
||||
pipeline-type:
|
||||
- 'bar'
|
||||
- 'baz'
|
||||
platform_axis:
|
||||
- 'centos6-64mdca'
|
||||
- 'centos7-64mdca'
|
||||
Loading…
Reference in New Issue