Merge "Inherit playbooks and modify job variance" into feature/zuulv3

This commit is contained in:
Jenkins 2017-02-15 21:47:31 +00:00 committed by Gerrit Code Review
commit 5d44129803
6 changed files with 382 additions and 104 deletions

View File

@ -27,6 +27,11 @@ from tests.base import BaseTestCase
class TestJob(BaseTestCase):
def setUp(self):
super(TestJob, self).setUp()
self.project = model.Project('project', None)
self.context = model.SourceContext(self.project, 'master', True)
@property
def job(self):
layout = model.Layout()
@ -54,6 +59,81 @@ class TestJob(BaseTestCase):
self.assertIsNotNone(self.job.voting)
def test_job_inheritance(self):
# This is standard job inheritance.
base_pre = model.PlaybookContext(self.context, 'base-pre')
base_run = model.PlaybookContext(self.context, 'base-run')
base_post = model.PlaybookContext(self.context, 'base-post')
base = model.Job('base')
base.timeout = 30
base.pre_run = [base_pre]
base.run = [base_run]
base.post_run = [base_post]
base.auth = dict(foo='bar', inherit=False)
py27 = model.Job('py27')
self.assertEqual(None, py27.timeout)
py27.inheritFrom(base)
self.assertEqual(30, py27.timeout)
self.assertEqual(['base-pre'],
[x.path for x in py27.pre_run])
self.assertEqual(['base-run'],
[x.path for x in py27.run])
self.assertEqual(['base-post'],
[x.path for x in py27.post_run])
self.assertEqual({}, py27.auth)
def test_job_variants(self):
# This simulates freezing a job.
py27_pre = model.PlaybookContext(self.context, 'py27-pre')
py27_run = model.PlaybookContext(self.context, 'py27-run')
py27_post = model.PlaybookContext(self.context, 'py27-post')
py27 = model.Job('py27')
py27.timeout = 30
py27.pre_run = [py27_pre]
py27.run = [py27_run]
py27.post_run = [py27_post]
auth = dict(foo='bar', inherit=False)
py27.auth = auth
job = py27.copy()
self.assertEqual(30, job.timeout)
# Apply the diablo variant
diablo = model.Job('py27')
diablo.timeout = 40
job.applyVariant(diablo)
self.assertEqual(40, job.timeout)
self.assertEqual(['py27-pre'],
[x.path for x in job.pre_run])
self.assertEqual(['py27-run'],
[x.path for x in job.run])
self.assertEqual(['py27-post'],
[x.path for x in job.post_run])
self.assertEqual(auth, job.auth)
# Set the job to final for the following checks
job.final = True
self.assertTrue(job.voting)
good_final = model.Job('py27')
good_final.voting = False
job.applyVariant(good_final)
self.assertFalse(job.voting)
bad_final = model.Job('py27')
bad_final.timeout = 600
with testtools.ExpectedException(
Exception,
"Unable to modify final job"):
job.applyVariant(bad_final)
def test_job_inheritance_configloader(self):
# TODO(jeblair): move this to a configloader test
layout = model.Layout()
pipeline = model.Pipeline('gate', layout)
@ -66,6 +146,8 @@ class TestJob(BaseTestCase):
'_source_context': context,
'name': 'base',
'timeout': 30,
'pre-run': 'base-pre',
'post-run': 'base-post',
'nodes': [{
'name': 'controller',
'image': 'base',
@ -76,6 +158,8 @@ class TestJob(BaseTestCase):
'_source_context': context,
'name': 'python27',
'parent': 'base',
'pre-run': 'py27-pre',
'post-run': 'py27-post',
'nodes': [{
'name': 'controller',
'image': 'new',
@ -89,6 +173,9 @@ class TestJob(BaseTestCase):
'branches': [
'stable/diablo'
],
'pre-run': 'py27-diablo-pre',
'run': 'py27-diablo',
'post-run': 'py27-diablo-post',
'nodes': [{
'name': 'controller',
'image': 'old',
@ -97,6 +184,17 @@ class TestJob(BaseTestCase):
})
layout.addJob(python27diablo)
python27essex = configloader.JobParser.fromYaml(layout, {
'_source_context': context,
'name': 'python27',
'branches': [
'stable/essex'
],
'pre-run': 'py27-essex-pre',
'post-run': 'py27-essex-post',
})
layout.addJob(python27essex)
project_config = configloader.ProjectParser.fromYaml(layout, {
'_source_context': context,
'name': 'project',
@ -117,6 +215,7 @@ class TestJob(BaseTestCase):
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
self.assertFalse(python27diablo.changeMatches(change))
self.assertFalse(python27essex.changeMatches(change))
item.freezeJobTree()
self.assertEqual(len(item.getJobs()), 1)
@ -126,6 +225,15 @@ class TestJob(BaseTestCase):
nodes = job.nodeset.getNodes()
self.assertEqual(len(nodes), 1)
self.assertEqual(nodes[0].image, 'new')
self.assertEqual([x.path for x in job.pre_run],
['playbooks/base-pre',
'playbooks/py27-pre'])
self.assertEqual([x.path for x in job.post_run],
['playbooks/py27-post',
'playbooks/base-post'])
self.assertEqual([x.path for x in job.run],
['playbooks/python27',
'playbooks/base'])
# Test diablo
change.branch = 'stable/diablo'
@ -135,6 +243,7 @@ class TestJob(BaseTestCase):
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
self.assertTrue(python27diablo.changeMatches(change))
self.assertFalse(python27essex.changeMatches(change))
item.freezeJobTree()
self.assertEqual(len(item.getJobs()), 1)
@ -144,6 +253,42 @@ class TestJob(BaseTestCase):
nodes = job.nodeset.getNodes()
self.assertEqual(len(nodes), 1)
self.assertEqual(nodes[0].image, 'old')
self.assertEqual([x.path for x in job.pre_run],
['playbooks/base-pre',
'playbooks/py27-pre',
'playbooks/py27-diablo-pre'])
self.assertEqual([x.path for x in job.post_run],
['playbooks/py27-diablo-post',
'playbooks/py27-post',
'playbooks/base-post'])
self.assertEqual([x.path for x in job.run],
['playbooks/py27-diablo']),
# Test essex
change.branch = 'stable/essex'
item = queue.enqueueChange(change)
item.current_build_set.layout = layout
self.assertTrue(base.changeMatches(change))
self.assertTrue(python27.changeMatches(change))
self.assertFalse(python27diablo.changeMatches(change))
self.assertTrue(python27essex.changeMatches(change))
item.freezeJobTree()
self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
self.assertEqual([x.path for x in job.pre_run],
['playbooks/base-pre',
'playbooks/py27-pre',
'playbooks/py27-essex-pre'])
self.assertEqual([x.path for x in job.post_run],
['playbooks/py27-essex-post',
'playbooks/py27-post',
'playbooks/base-post'])
self.assertEqual([x.path for x in job.run],
['playbooks/python27',
'playbooks/base'])
def test_job_auth_inheritance(self):
layout = model.Layout()

View File

@ -35,9 +35,15 @@ class AbstractChangeMatcher(object):
def copy(self):
return self.__class__(self._regex)
def __deepcopy__(self, memo):
return self.copy()
def __eq__(self, other):
return str(self) == str(other)
def __ne__(self, other):
return not self.__eq__(other)
def __str__(self):
return '{%s:%s}' % (self.__class__.__name__, self._regex)

View File

@ -103,27 +103,66 @@ class JobParser(object):
'attempts': int,
'pre-run': to_list(str),
'post-run': to_list(str),
'run': str,
'_source_context': model.SourceContext,
}
return vs.Schema(job)
simple_attributes = [
'timeout',
'workspace',
'voting',
'hold-following-changes',
'mutex',
'attempts',
'failure-message',
'success-message',
'failure-url',
'success-url',
]
@staticmethod
def fromYaml(layout, conf):
JobParser.getSchema()(conf)
# NB: The default detection system in the Job class requires
# that we always assign values directly rather than modifying
# them (e.g., "job.run = ..." rather than
# "job.run.append(...)").
job = model.Job(conf['name'])
job.source_context = conf.get('_source_context')
if 'auth' in conf:
job.auth = conf.get('auth')
if 'parent' in conf:
parent = layout.getJob(conf['parent'])
job.inheritFrom(parent, 'parent while parsing')
job.timeout = conf.get('timeout', job.timeout)
job.workspace = conf.get('workspace', job.workspace)
job.voting = conf.get('voting', True)
job.hold_following_changes = conf.get('hold-following-changes', False)
job.mutex = conf.get('mutex', None)
job.attempts = conf.get('attempts', 3)
job.inheritFrom(parent)
for pre_run_name in as_list(conf.get('pre-run')):
full_pre_run_name = os.path.join('playbooks', pre_run_name)
pre_run = model.PlaybookContext(job.source_context,
full_pre_run_name)
job.pre_run = job.pre_run + (pre_run,)
for post_run_name in as_list(conf.get('post-run')):
full_post_run_name = os.path.join('playbooks', post_run_name)
post_run = model.PlaybookContext(job.source_context,
full_post_run_name)
job.post_run = (post_run,) + job.post_run
if 'run' in conf:
run_name = os.path.join('playbooks', conf['run'])
run = model.PlaybookContext(job.source_context, run_name)
job.run = (run,)
else:
run_name = os.path.join('playbooks', job.name)
run = model.PlaybookContext(job.source_context, run_name)
job.implied_run = (run,) + job.implied_run
for k in JobParser.simple_attributes:
a = k.replace('-', '_')
if k in conf:
setattr(job, a, conf[k])
if 'nodes' in conf:
conf_nodes = conf['nodes']
if isinstance(conf_nodes, six.string_types):
@ -140,37 +179,8 @@ class JobParser(object):
if tags:
# Tags are merged via a union rather than a
# destructive copy because they are intended to
# accumulate onto any previously applied tags from
# metajobs.
# accumulate onto any previously applied tags.
job.tags = job.tags.union(set(tags))
# The source attribute and playbook info may not be
# overridden -- they are always supplied by the config loader.
# They correspond to the Project instance of the repo where it
# originated, and the branch name.
job.source_context = conf.get('_source_context')
pre_run_name = conf.get('pre-run')
# Append the pre-run command
if pre_run_name:
pre_run_name = os.path.join('playbooks', pre_run_name)
pre_run = model.PlaybookContext(job.source_context,
pre_run_name)
job.pre_run.append(pre_run)
# Prepend the post-run command
post_run_name = conf.get('post-run')
if post_run_name:
post_run_name = os.path.join('playbooks', post_run_name)
post_run = model.PlaybookContext(job.source_context,
post_run_name)
job.post_run.insert(0, post_run)
# Set the run command
run_name = job.name
run_name = os.path.join('playbooks', run_name)
run = model.PlaybookContext(job.source_context, run_name)
job.run = run
job.failure_message = conf.get('failure-message', job.failure_message)
job.success_message = conf.get('success-message', job.success_message)
job.failure_url = conf.get('failure-url', job.failure_url)
job.success_url = conf.get('success-url', job.success_url)
# If the definition for this job came from a project repo,
# implicitly apply a branch matcher for the branch it was on.
@ -240,7 +250,8 @@ class ProjectTemplateParser(object):
tree = model.JobTree(None)
for conf_job in conf:
if isinstance(conf_job, six.string_types):
tree.addJob(model.Job(conf_job))
job = model.Job(conf_job)
tree.addJob(job)
elif isinstance(conf_job, dict):
# A dictionary in a job tree may override params, or
# be the root of a sub job tree, or both.
@ -252,8 +263,9 @@ class ProjectTemplateParser(object):
attrs['_source_context'] = source_context
subtree = tree.addJob(JobParser.fromYaml(layout, attrs))
else:
# Not overriding, so get existing job
subtree = tree.addJob(layout.getJob(jobname))
# Not overriding, so add a blank job
job = model.Job(jobname)
subtree = tree.addJob(job)
if jobs:
# This is the root of a sub tree
@ -313,8 +325,7 @@ class ProjectParser(object):
pipeline_defined = True
template_pipeline = template.pipelines[pipeline.name]
project_pipeline.job_tree.inheritFrom(
template_pipeline.job_tree,
'job tree while parsing')
template_pipeline.job_tree)
if template_pipeline.queue_name:
queue_name = template_pipeline.queue_name
if queue_name:

View File

@ -378,7 +378,7 @@ class LaunchClient(object):
params['projects'] = []
if job.name != 'noop':
params['playbook'] = job.run.toDict()
params['playbooks'] = [x.toDict() for x in job.run]
params['pre_playbooks'] = [x.toDict() for x in job.pre_run]
params['post_playbooks'] = [x.toDict() for x in job.post_run]

View File

@ -88,9 +88,8 @@ class JobDir(object):
self.known_hosts = os.path.join(self.ansible_root, 'known_hosts')
self.inventory = os.path.join(self.ansible_root, 'inventory')
self.vars = os.path.join(self.ansible_root, 'vars.yaml')
self.playbook_root = os.path.join(self.ansible_root, 'playbook')
os.makedirs(self.playbook_root)
self.playbook = JobDirPlaybook(self.playbook_root)
self.playbooks = [] # The list of candidate playbooks
self.playbook = None # A pointer to the candidate we have chosen
self.pre_playbooks = []
self.post_playbooks = []
self.config = os.path.join(self.ansible_root, 'ansible.cfg')
@ -112,6 +111,14 @@ class JobDir(object):
self.post_playbooks.append(playbook)
return playbook
def addPlaybook(self):
count = len(self.playbooks)
root = os.path.join(self.ansible_root, 'playbook_%i' % (count,))
os.makedirs(root)
playbook = JobDirPlaybook(root)
self.playbooks.append(playbook)
return playbook
def cleanup(self):
if not self.keep:
shutil.rmtree(self.root)
@ -573,26 +580,38 @@ class AnsibleJob(object):
hosts.append((node['name'], dict(ansible_connection='local')))
return hosts
def findPlaybook(self, path):
def findPlaybook(self, path, required=False):
for ext in ['.yaml', '.yml']:
fn = path + ext
if os.path.exists(fn):
return fn
raise Exception("Unable to find playbook %s" % path)
if required:
raise Exception("Unable to find playbook %s" % path)
return None
def preparePlaybookRepos(self, args):
for playbook in args['pre_playbooks']:
jobdir_playbook = self.jobdir.addPrePlaybook()
self.preparePlaybookRepo(jobdir_playbook, playbook, args)
self.preparePlaybookRepo(jobdir_playbook, playbook,
args, main=False)
jobdir_playbook = self.jobdir.playbook
self.preparePlaybookRepo(jobdir_playbook, args['playbook'], args)
for playbook in args['playbooks']:
jobdir_playbook = self.jobdir.addPlaybook()
self.preparePlaybookRepo(jobdir_playbook, playbook,
args, main=True)
if jobdir_playbook.path is not None:
self.jobdir.playbook = jobdir_playbook
break
if self.jobdir.playbook is None:
raise Exception("No valid playbook found")
for playbook in args['post_playbooks']:
jobdir_playbook = self.jobdir.addPostPlaybook()
self.preparePlaybookRepo(jobdir_playbook, playbook, args)
self.preparePlaybookRepo(jobdir_playbook, playbook,
args, main=False)
def preparePlaybookRepo(self, jobdir_playbook, playbook, args):
def preparePlaybookRepo(self, jobdir_playbook, playbook, args, main):
self.log.debug("Prepare playbook repo for %s" % (playbook,))
# Check out the playbook repo if needed and set the path to
# the playbook that should be run.
jobdir_playbook.secure = playbook['secure']
@ -612,7 +631,7 @@ class AnsibleJob(object):
path = os.path.join(self.jobdir.git_root,
project.name,
playbook['path'])
jobdir_playbook.path = self.findPlaybook(path)
jobdir_playbook.path = self.findPlaybook(path, main)
return
# The playbook repo is either a config repo, or it isn't in
# the stack of changes we are testing, so check out the branch
@ -624,7 +643,7 @@ class AnsibleJob(object):
path = os.path.join(jobdir_playbook.root,
project.name,
playbook['path'])
jobdir_playbook.path = self.findPlaybook(path)
jobdir_playbook.path = self.findPlaybook(path, main)
def prepareAnsibleFiles(self, args):
with open(self.jobdir.inventory, 'w') as inventory:

View File

@ -541,6 +541,12 @@ class SourceContext(object):
self.branch,
self.secure)
def __deepcopy__(self, memo):
return self.copy()
def copy(self):
return self.__class__(self.project, self.branch, self.secure)
def __ne__(self, other):
return not self.__eq__(other)
@ -589,20 +595,20 @@ class PlaybookContext(object):
class Job(object):
"""A Job represents the defintion of actions to perform."""
"""A Job represents the defintion of actions to perform.
NB: Do not modify attributes of this class, set them directly
(e.g., "job.run = ..." rather than "job.run.append(...)").
"""
def __init__(self, name):
self.attributes = dict(
timeout=None,
# variables={},
nodeset=NodeSet(),
auth={},
workspace=None,
pre_run=[],
post_run=[],
run=None,
voting=None,
hold_following_changes=None,
# These attributes may override even the final form of a job
# in the context of a project-pipeline. They can not affect
# the execution of the job, but only whether the job is run
# and how it is reported.
self.context_attributes = dict(
voting=True,
hold_following_changes=False,
failure_message=None,
success_message=None,
failure_url=None,
@ -612,16 +618,44 @@ class Job(object):
branch_matcher=None,
file_matcher=None,
irrelevant_file_matcher=None, # skip-if
tags=set(),
mutex=None,
attempts=3,
source_context=None,
inheritance_path=[],
tags=frozenset(),
)
# These attributes affect how the job is actually run and more
# care must be taken when overriding them. If a job is
# declared "final", these may not be overriden in a
# project-pipeline.
self.execution_attributes = dict(
timeout=None,
# variables={},
nodeset=NodeSet(),
auth={},
workspace=None,
pre_run=(),
post_run=(),
run=(),
implied_run=(),
mutex=None,
attempts=3,
final=False,
)
# These are generally internal attributes which are not
# accessible via configuration.
self.other_attributes = dict(
name=None,
source_context=None,
inheritance_path=(),
)
self.inheritable_attributes = {}
self.inheritable_attributes.update(self.context_attributes)
self.inheritable_attributes.update(self.execution_attributes)
self.attributes = {}
self.attributes.update(self.inheritable_attributes)
self.attributes.update(self.other_attributes)
self.name = name
for k, v in self.attributes.items():
setattr(self, k, v)
def __ne__(self, other):
return not self.__eq__(other)
@ -647,24 +681,82 @@ class Job(object):
self.branch_matcher,
self.source_context)
def inheritFrom(self, other, comment='unknown'):
def __getattr__(self, name):
v = self.__dict__.get(name)
if v is None:
return copy.deepcopy(self.attributes[name])
return v
def _get(self, name):
return self.__dict__.get(name)
def setRun(self):
if not self.run:
self.run = self.implied_run
def inheritFrom(self, other):
"""Copy the inheritable attributes which have been set on the other
job to this job."""
if not isinstance(other, Job):
raise Exception("Job unable to inherit from %s" % (other,))
do_not_inherit = set()
if other.auth and not other.auth.get('inherit'):
do_not_inherit.add('auth')
# copy all attributes
for k in self.inheritable_attributes:
if (other._get(k) is not None and k not in do_not_inherit):
setattr(self, k, copy.deepcopy(getattr(other, k)))
msg = 'inherit from %s' % (repr(other),)
self.inheritance_path = other.inheritance_path + (msg,)
def copy(self):
job = Job(self.name)
for k in self.attributes:
if self._get(k) is not None:
setattr(job, k, copy.deepcopy(self._get(k)))
return job
def applyVariant(self, other):
"""Copy the attributes which have been set on the other job to this
job."""
if not isinstance(other, Job):
raise Exception("Job unable to inherit from %s" % (other,))
self.inheritance_path.extend(other.inheritance_path)
self.inheritance_path.append('%s %s' % (repr(other), comment))
for k, v in self.attributes.items():
if (getattr(other, k) != v and k not in
set(['auth', 'pre_run', 'post_run', 'inheritance_path'])):
setattr(self, k, getattr(other, k))
# Inherit auth only if explicitly allowed
if other.auth and 'inherit' in other.auth and other.auth['inherit']:
setattr(self, 'auth', getattr(other, 'auth'))
# Pre and post run are lists; make a copy
self.pre_run = other.pre_run + self.pre_run
self.post_run = self.post_run + other.post_run
for k in self.execution_attributes:
if (other._get(k) is not None and
k not in set(['final'])):
if self.final:
raise Exception("Unable to modify final job %s attribute "
"%s=%s with variant %s" % (
repr(self), k, other._get(k),
repr(other)))
if k not in set(['pre_run', 'post_run']):
setattr(self, k, copy.deepcopy(other._get(k)))
# Don't set final above so that we don't trip an error halfway
# through assignment.
if other.final != self.attributes['final']:
self.final = other.final
if other._get('pre_run') is not None:
self.pre_run = self.pre_run + other.pre_run
if other._get('post_run') is not None:
self.post_run = other.post_run + self.post_run
for k in self.context_attributes:
if (other._get(k) is not None and
k not in set(['tags'])):
setattr(self, k, copy.deepcopy(other._get(k)))
if other._get('tags') is not None:
self.tags = self.tags.union(other.tags)
msg = 'apply variant %s' % (repr(other),)
self.inheritance_path = self.inheritance_path + (msg,)
def changeMatches(self, change):
if self.branch_matcher and not self.branch_matcher.matches(change):
@ -720,16 +812,18 @@ class JobTree(object):
return ret
return None
def inheritFrom(self, other, comment='unknown'):
def inheritFrom(self, other):
if other.job:
self.job = Job(other.job.name)
self.job.inheritFrom(other.job, comment)
if not self.job:
self.job = other.job.copy()
else:
self.job.applyVariant(other.job)
for other_tree in other.job_trees:
this_tree = self.getJobTreeForJob(other_tree.job)
if not this_tree:
this_tree = JobTree(None)
self.job_trees.append(this_tree)
this_tree.inheritFrom(other_tree, comment)
this_tree.inheritFrom(other_tree)
class Build(object):
@ -1994,25 +2088,28 @@ class Layout(object):
job = tree.job
if not job.changeMatches(change):
continue
frozen_job = Job(job.name)
frozen_tree = JobTree(frozen_job)
inherited = set()
frozen_job = None
matched = False
for variant in self.getJobs(job.name):
if variant.changeMatches(change):
if variant not in inherited:
frozen_job.inheritFrom(variant,
'variant while freezing')
inherited.add(variant)
if not inherited:
if frozen_job is None:
frozen_job = variant.copy()
frozen_job.setRun()
else:
frozen_job.applyVariant(variant)
matched = True
if not matched:
# A change must match at least one defined job variant
# (that is to say that it must match more than just
# the job that is defined in the tree).
continue
if job not in inherited:
# Only update from the job in the tree if it is
# unique, otherwise we might unset an attribute we
# have overloaded.
frozen_job.inheritFrom(job, 'tree job while freezing')
# If the job does not allow auth inheritance, do not allow
# the project-pipeline variant to update its execution
# attributes.
if frozen_job.auth and not frozen_job.auth.get('inherit'):
frozen_job.final = True
frozen_job.applyVariant(job)
frozen_tree = JobTree(frozen_job)
parent.job_trees.append(frozen_tree)
self._createJobTree(change, tree.job_trees, frozen_tree)