Inherit playbooks and modify job variance
An earlier change dealt with inheritance for pre and post playbooks; they are nested so that parent job pre and post playbooks run first and last respectively. As for the actual playbook, since it's implied by the job name, it's not clear whether it should be overidden or not. We could drop that and say that if you specify a 'run' attribute, it means you want to set the playbook for a job, but if you omit it, you want to use the parent's playbook. However, we could keep the implied playbook behavior by making the 'run' attribute a list and adding a playbook context to the list each time a job is inherited. Then the launcher can walk the list in order and the first playbook it finds, it runs. This is what is implemented here. However, we need to restrict playbooks or other execution-related job attributes from being overidden by out-of-repo variants (such as the implicit variant which is created by every entry in a project-pipeline). To do this, we make more of a distinction between inheritance and variance, implementing each with its own method on Job. This way we can better control when certain attributes are allowed to be set. The 'final' job attribute is added to indicate that a job should not accept any further modifications to execution-related attributes. The attribute storage in Job is altered so that each Job object explicitly stores whether an attribute was set on it. This makes it easier to start with a job and apply only the specified attributes of each variant in turn. Default values are still handled. Essentially, each "job" appearance in the configuration will create a new Job entry with exactly those attributes (with the exception that a job where "parent" is set will first copy attributes which are explicitly set on its parent). When a job is frozen after an item is enqueued, the first matching job is copied, and each subsequent matching job is applied as a varient. When that is completed, if the job has un-inheritable auth information, it is set as final, and then the project-pipeline variant is applied. New tests are added to exercise the new methods on Job. Change-Id: Iaf6d661a7bd0085e55bc301f83fe158fd0a70166
This commit is contained in:
parent
f10f985b2f
commit
a7f51ca625
@ -27,6 +27,11 @@ from tests.base import BaseTestCase
|
||||
|
||||
class TestJob(BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestJob, self).setUp()
|
||||
self.project = model.Project('project', None)
|
||||
self.context = model.SourceContext(self.project, 'master', True)
|
||||
|
||||
@property
|
||||
def job(self):
|
||||
layout = model.Layout()
|
||||
@ -54,6 +59,81 @@ class TestJob(BaseTestCase):
|
||||
self.assertIsNotNone(self.job.voting)
|
||||
|
||||
def test_job_inheritance(self):
|
||||
# This is standard job inheritance.
|
||||
|
||||
base_pre = model.PlaybookContext(self.context, 'base-pre')
|
||||
base_run = model.PlaybookContext(self.context, 'base-run')
|
||||
base_post = model.PlaybookContext(self.context, 'base-post')
|
||||
|
||||
base = model.Job('base')
|
||||
base.timeout = 30
|
||||
base.pre_run = [base_pre]
|
||||
base.run = [base_run]
|
||||
base.post_run = [base_post]
|
||||
base.auth = dict(foo='bar', inherit=False)
|
||||
|
||||
py27 = model.Job('py27')
|
||||
self.assertEqual(None, py27.timeout)
|
||||
py27.inheritFrom(base)
|
||||
self.assertEqual(30, py27.timeout)
|
||||
self.assertEqual(['base-pre'],
|
||||
[x.path for x in py27.pre_run])
|
||||
self.assertEqual(['base-run'],
|
||||
[x.path for x in py27.run])
|
||||
self.assertEqual(['base-post'],
|
||||
[x.path for x in py27.post_run])
|
||||
self.assertEqual({}, py27.auth)
|
||||
|
||||
def test_job_variants(self):
|
||||
# This simulates freezing a job.
|
||||
|
||||
py27_pre = model.PlaybookContext(self.context, 'py27-pre')
|
||||
py27_run = model.PlaybookContext(self.context, 'py27-run')
|
||||
py27_post = model.PlaybookContext(self.context, 'py27-post')
|
||||
|
||||
py27 = model.Job('py27')
|
||||
py27.timeout = 30
|
||||
py27.pre_run = [py27_pre]
|
||||
py27.run = [py27_run]
|
||||
py27.post_run = [py27_post]
|
||||
auth = dict(foo='bar', inherit=False)
|
||||
py27.auth = auth
|
||||
|
||||
job = py27.copy()
|
||||
self.assertEqual(30, job.timeout)
|
||||
|
||||
# Apply the diablo variant
|
||||
diablo = model.Job('py27')
|
||||
diablo.timeout = 40
|
||||
job.applyVariant(diablo)
|
||||
|
||||
self.assertEqual(40, job.timeout)
|
||||
self.assertEqual(['py27-pre'],
|
||||
[x.path for x in job.pre_run])
|
||||
self.assertEqual(['py27-run'],
|
||||
[x.path for x in job.run])
|
||||
self.assertEqual(['py27-post'],
|
||||
[x.path for x in job.post_run])
|
||||
self.assertEqual(auth, job.auth)
|
||||
|
||||
# Set the job to final for the following checks
|
||||
job.final = True
|
||||
self.assertTrue(job.voting)
|
||||
|
||||
good_final = model.Job('py27')
|
||||
good_final.voting = False
|
||||
job.applyVariant(good_final)
|
||||
self.assertFalse(job.voting)
|
||||
|
||||
bad_final = model.Job('py27')
|
||||
bad_final.timeout = 600
|
||||
with testtools.ExpectedException(
|
||||
Exception,
|
||||
"Unable to modify final job"):
|
||||
job.applyVariant(bad_final)
|
||||
|
||||
def test_job_inheritance_configloader(self):
|
||||
# TODO(jeblair): move this to a configloader test
|
||||
layout = model.Layout()
|
||||
|
||||
pipeline = model.Pipeline('gate', layout)
|
||||
@ -66,6 +146,8 @@ class TestJob(BaseTestCase):
|
||||
'_source_context': context,
|
||||
'name': 'base',
|
||||
'timeout': 30,
|
||||
'pre-run': 'base-pre',
|
||||
'post-run': 'base-post',
|
||||
'nodes': [{
|
||||
'name': 'controller',
|
||||
'image': 'base',
|
||||
@ -76,6 +158,8 @@ class TestJob(BaseTestCase):
|
||||
'_source_context': context,
|
||||
'name': 'python27',
|
||||
'parent': 'base',
|
||||
'pre-run': 'py27-pre',
|
||||
'post-run': 'py27-post',
|
||||
'nodes': [{
|
||||
'name': 'controller',
|
||||
'image': 'new',
|
||||
@ -89,6 +173,9 @@ class TestJob(BaseTestCase):
|
||||
'branches': [
|
||||
'stable/diablo'
|
||||
],
|
||||
'pre-run': 'py27-diablo-pre',
|
||||
'run': 'py27-diablo',
|
||||
'post-run': 'py27-diablo-post',
|
||||
'nodes': [{
|
||||
'name': 'controller',
|
||||
'image': 'old',
|
||||
@ -97,6 +184,17 @@ class TestJob(BaseTestCase):
|
||||
})
|
||||
layout.addJob(python27diablo)
|
||||
|
||||
python27essex = configloader.JobParser.fromYaml(layout, {
|
||||
'_source_context': context,
|
||||
'name': 'python27',
|
||||
'branches': [
|
||||
'stable/essex'
|
||||
],
|
||||
'pre-run': 'py27-essex-pre',
|
||||
'post-run': 'py27-essex-post',
|
||||
})
|
||||
layout.addJob(python27essex)
|
||||
|
||||
project_config = configloader.ProjectParser.fromYaml(layout, {
|
||||
'_source_context': context,
|
||||
'name': 'project',
|
||||
@ -117,6 +215,7 @@ class TestJob(BaseTestCase):
|
||||
self.assertTrue(base.changeMatches(change))
|
||||
self.assertTrue(python27.changeMatches(change))
|
||||
self.assertFalse(python27diablo.changeMatches(change))
|
||||
self.assertFalse(python27essex.changeMatches(change))
|
||||
|
||||
item.freezeJobTree()
|
||||
self.assertEqual(len(item.getJobs()), 1)
|
||||
@ -126,6 +225,15 @@ class TestJob(BaseTestCase):
|
||||
nodes = job.nodeset.getNodes()
|
||||
self.assertEqual(len(nodes), 1)
|
||||
self.assertEqual(nodes[0].image, 'new')
|
||||
self.assertEqual([x.path for x in job.pre_run],
|
||||
['playbooks/base-pre',
|
||||
'playbooks/py27-pre'])
|
||||
self.assertEqual([x.path for x in job.post_run],
|
||||
['playbooks/py27-post',
|
||||
'playbooks/base-post'])
|
||||
self.assertEqual([x.path for x in job.run],
|
||||
['playbooks/python27',
|
||||
'playbooks/base'])
|
||||
|
||||
# Test diablo
|
||||
change.branch = 'stable/diablo'
|
||||
@ -135,6 +243,7 @@ class TestJob(BaseTestCase):
|
||||
self.assertTrue(base.changeMatches(change))
|
||||
self.assertTrue(python27.changeMatches(change))
|
||||
self.assertTrue(python27diablo.changeMatches(change))
|
||||
self.assertFalse(python27essex.changeMatches(change))
|
||||
|
||||
item.freezeJobTree()
|
||||
self.assertEqual(len(item.getJobs()), 1)
|
||||
@ -144,6 +253,42 @@ class TestJob(BaseTestCase):
|
||||
nodes = job.nodeset.getNodes()
|
||||
self.assertEqual(len(nodes), 1)
|
||||
self.assertEqual(nodes[0].image, 'old')
|
||||
self.assertEqual([x.path for x in job.pre_run],
|
||||
['playbooks/base-pre',
|
||||
'playbooks/py27-pre',
|
||||
'playbooks/py27-diablo-pre'])
|
||||
self.assertEqual([x.path for x in job.post_run],
|
||||
['playbooks/py27-diablo-post',
|
||||
'playbooks/py27-post',
|
||||
'playbooks/base-post'])
|
||||
self.assertEqual([x.path for x in job.run],
|
||||
['playbooks/py27-diablo']),
|
||||
|
||||
# Test essex
|
||||
change.branch = 'stable/essex'
|
||||
item = queue.enqueueChange(change)
|
||||
item.current_build_set.layout = layout
|
||||
|
||||
self.assertTrue(base.changeMatches(change))
|
||||
self.assertTrue(python27.changeMatches(change))
|
||||
self.assertFalse(python27diablo.changeMatches(change))
|
||||
self.assertTrue(python27essex.changeMatches(change))
|
||||
|
||||
item.freezeJobTree()
|
||||
self.assertEqual(len(item.getJobs()), 1)
|
||||
job = item.getJobs()[0]
|
||||
self.assertEqual(job.name, 'python27')
|
||||
self.assertEqual([x.path for x in job.pre_run],
|
||||
['playbooks/base-pre',
|
||||
'playbooks/py27-pre',
|
||||
'playbooks/py27-essex-pre'])
|
||||
self.assertEqual([x.path for x in job.post_run],
|
||||
['playbooks/py27-essex-post',
|
||||
'playbooks/py27-post',
|
||||
'playbooks/base-post'])
|
||||
self.assertEqual([x.path for x in job.run],
|
||||
['playbooks/python27',
|
||||
'playbooks/base'])
|
||||
|
||||
def test_job_auth_inheritance(self):
|
||||
layout = model.Layout()
|
||||
|
@ -35,9 +35,15 @@ class AbstractChangeMatcher(object):
|
||||
def copy(self):
|
||||
return self.__class__(self._regex)
|
||||
|
||||
def __deepcopy__(self, memo):
|
||||
return self.copy()
|
||||
|
||||
def __eq__(self, other):
|
||||
return str(self) == str(other)
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self.__eq__(other)
|
||||
|
||||
def __str__(self):
|
||||
return '{%s:%s}' % (self.__class__.__name__, self._regex)
|
||||
|
||||
|
@ -103,27 +103,66 @@ class JobParser(object):
|
||||
'attempts': int,
|
||||
'pre-run': to_list(str),
|
||||
'post-run': to_list(str),
|
||||
'run': str,
|
||||
'_source_context': model.SourceContext,
|
||||
}
|
||||
|
||||
return vs.Schema(job)
|
||||
|
||||
simple_attributes = [
|
||||
'timeout',
|
||||
'workspace',
|
||||
'voting',
|
||||
'hold-following-changes',
|
||||
'mutex',
|
||||
'attempts',
|
||||
'failure-message',
|
||||
'success-message',
|
||||
'failure-url',
|
||||
'success-url',
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def fromYaml(layout, conf):
|
||||
JobParser.getSchema()(conf)
|
||||
|
||||
# NB: The default detection system in the Job class requires
|
||||
# that we always assign values directly rather than modifying
|
||||
# them (e.g., "job.run = ..." rather than
|
||||
# "job.run.append(...)").
|
||||
|
||||
job = model.Job(conf['name'])
|
||||
job.source_context = conf.get('_source_context')
|
||||
if 'auth' in conf:
|
||||
job.auth = conf.get('auth')
|
||||
|
||||
if 'parent' in conf:
|
||||
parent = layout.getJob(conf['parent'])
|
||||
job.inheritFrom(parent, 'parent while parsing')
|
||||
job.timeout = conf.get('timeout', job.timeout)
|
||||
job.workspace = conf.get('workspace', job.workspace)
|
||||
job.voting = conf.get('voting', True)
|
||||
job.hold_following_changes = conf.get('hold-following-changes', False)
|
||||
job.mutex = conf.get('mutex', None)
|
||||
job.attempts = conf.get('attempts', 3)
|
||||
job.inheritFrom(parent)
|
||||
|
||||
for pre_run_name in as_list(conf.get('pre-run')):
|
||||
full_pre_run_name = os.path.join('playbooks', pre_run_name)
|
||||
pre_run = model.PlaybookContext(job.source_context,
|
||||
full_pre_run_name)
|
||||
job.pre_run = job.pre_run + (pre_run,)
|
||||
for post_run_name in as_list(conf.get('post-run')):
|
||||
full_post_run_name = os.path.join('playbooks', post_run_name)
|
||||
post_run = model.PlaybookContext(job.source_context,
|
||||
full_post_run_name)
|
||||
job.post_run = (post_run,) + job.post_run
|
||||
if 'run' in conf:
|
||||
run_name = os.path.join('playbooks', conf['run'])
|
||||
run = model.PlaybookContext(job.source_context, run_name)
|
||||
job.run = (run,)
|
||||
else:
|
||||
run_name = os.path.join('playbooks', job.name)
|
||||
run = model.PlaybookContext(job.source_context, run_name)
|
||||
job.implied_run = (run,) + job.implied_run
|
||||
|
||||
for k in JobParser.simple_attributes:
|
||||
a = k.replace('-', '_')
|
||||
if k in conf:
|
||||
setattr(job, a, conf[k])
|
||||
if 'nodes' in conf:
|
||||
conf_nodes = conf['nodes']
|
||||
if isinstance(conf_nodes, six.string_types):
|
||||
@ -140,37 +179,8 @@ class JobParser(object):
|
||||
if tags:
|
||||
# Tags are merged via a union rather than a
|
||||
# destructive copy because they are intended to
|
||||
# accumulate onto any previously applied tags from
|
||||
# metajobs.
|
||||
# accumulate onto any previously applied tags.
|
||||
job.tags = job.tags.union(set(tags))
|
||||
# The source attribute and playbook info may not be
|
||||
# overridden -- they are always supplied by the config loader.
|
||||
# They correspond to the Project instance of the repo where it
|
||||
# originated, and the branch name.
|
||||
job.source_context = conf.get('_source_context')
|
||||
pre_run_name = conf.get('pre-run')
|
||||
# Append the pre-run command
|
||||
if pre_run_name:
|
||||
pre_run_name = os.path.join('playbooks', pre_run_name)
|
||||
pre_run = model.PlaybookContext(job.source_context,
|
||||
pre_run_name)
|
||||
job.pre_run.append(pre_run)
|
||||
# Prepend the post-run command
|
||||
post_run_name = conf.get('post-run')
|
||||
if post_run_name:
|
||||
post_run_name = os.path.join('playbooks', post_run_name)
|
||||
post_run = model.PlaybookContext(job.source_context,
|
||||
post_run_name)
|
||||
job.post_run.insert(0, post_run)
|
||||
# Set the run command
|
||||
run_name = job.name
|
||||
run_name = os.path.join('playbooks', run_name)
|
||||
run = model.PlaybookContext(job.source_context, run_name)
|
||||
job.run = run
|
||||
job.failure_message = conf.get('failure-message', job.failure_message)
|
||||
job.success_message = conf.get('success-message', job.success_message)
|
||||
job.failure_url = conf.get('failure-url', job.failure_url)
|
||||
job.success_url = conf.get('success-url', job.success_url)
|
||||
|
||||
# If the definition for this job came from a project repo,
|
||||
# implicitly apply a branch matcher for the branch it was on.
|
||||
@ -240,7 +250,8 @@ class ProjectTemplateParser(object):
|
||||
tree = model.JobTree(None)
|
||||
for conf_job in conf:
|
||||
if isinstance(conf_job, six.string_types):
|
||||
tree.addJob(model.Job(conf_job))
|
||||
job = model.Job(conf_job)
|
||||
tree.addJob(job)
|
||||
elif isinstance(conf_job, dict):
|
||||
# A dictionary in a job tree may override params, or
|
||||
# be the root of a sub job tree, or both.
|
||||
@ -252,8 +263,9 @@ class ProjectTemplateParser(object):
|
||||
attrs['_source_context'] = source_context
|
||||
subtree = tree.addJob(JobParser.fromYaml(layout, attrs))
|
||||
else:
|
||||
# Not overriding, so get existing job
|
||||
subtree = tree.addJob(layout.getJob(jobname))
|
||||
# Not overriding, so add a blank job
|
||||
job = model.Job(jobname)
|
||||
subtree = tree.addJob(job)
|
||||
|
||||
if jobs:
|
||||
# This is the root of a sub tree
|
||||
@ -313,8 +325,7 @@ class ProjectParser(object):
|
||||
pipeline_defined = True
|
||||
template_pipeline = template.pipelines[pipeline.name]
|
||||
project_pipeline.job_tree.inheritFrom(
|
||||
template_pipeline.job_tree,
|
||||
'job tree while parsing')
|
||||
template_pipeline.job_tree)
|
||||
if template_pipeline.queue_name:
|
||||
queue_name = template_pipeline.queue_name
|
||||
if queue_name:
|
||||
|
@ -374,7 +374,7 @@ class LaunchClient(object):
|
||||
params['projects'] = []
|
||||
|
||||
if job.name != 'noop':
|
||||
params['playbook'] = job.run.toDict()
|
||||
params['playbooks'] = [x.toDict() for x in job.run]
|
||||
params['pre_playbooks'] = [x.toDict() for x in job.pre_run]
|
||||
params['post_playbooks'] = [x.toDict() for x in job.post_run]
|
||||
|
||||
|
@ -84,9 +84,8 @@ class JobDir(object):
|
||||
self.known_hosts = os.path.join(self.ansible_root, 'known_hosts')
|
||||
self.inventory = os.path.join(self.ansible_root, 'inventory')
|
||||
self.vars = os.path.join(self.ansible_root, 'vars.yaml')
|
||||
self.playbook_root = os.path.join(self.ansible_root, 'playbook')
|
||||
os.makedirs(self.playbook_root)
|
||||
self.playbook = JobDirPlaybook(self.playbook_root)
|
||||
self.playbooks = [] # The list of candidate playbooks
|
||||
self.playbook = None # A pointer to the candidate we have chosen
|
||||
self.pre_playbooks = []
|
||||
self.post_playbooks = []
|
||||
self.config = os.path.join(self.ansible_root, 'ansible.cfg')
|
||||
@ -108,6 +107,14 @@ class JobDir(object):
|
||||
self.post_playbooks.append(playbook)
|
||||
return playbook
|
||||
|
||||
def addPlaybook(self):
|
||||
count = len(self.playbooks)
|
||||
root = os.path.join(self.ansible_root, 'playbook_%i' % (count,))
|
||||
os.makedirs(root)
|
||||
playbook = JobDirPlaybook(root)
|
||||
self.playbooks.append(playbook)
|
||||
return playbook
|
||||
|
||||
def cleanup(self):
|
||||
if not self.keep:
|
||||
shutil.rmtree(self.root)
|
||||
@ -563,26 +570,38 @@ class AnsibleJob(object):
|
||||
hosts.append((node['name'], dict(ansible_connection='local')))
|
||||
return hosts
|
||||
|
||||
def findPlaybook(self, path):
|
||||
def findPlaybook(self, path, required=False):
|
||||
for ext in ['.yaml', '.yml']:
|
||||
fn = path + ext
|
||||
if os.path.exists(fn):
|
||||
return fn
|
||||
raise Exception("Unable to find playbook %s" % path)
|
||||
if required:
|
||||
raise Exception("Unable to find playbook %s" % path)
|
||||
return None
|
||||
|
||||
def preparePlaybookRepos(self, args):
|
||||
for playbook in args['pre_playbooks']:
|
||||
jobdir_playbook = self.jobdir.addPrePlaybook()
|
||||
self.preparePlaybookRepo(jobdir_playbook, playbook, args)
|
||||
self.preparePlaybookRepo(jobdir_playbook, playbook,
|
||||
args, main=False)
|
||||
|
||||
jobdir_playbook = self.jobdir.playbook
|
||||
self.preparePlaybookRepo(jobdir_playbook, args['playbook'], args)
|
||||
for playbook in args['playbooks']:
|
||||
jobdir_playbook = self.jobdir.addPlaybook()
|
||||
self.preparePlaybookRepo(jobdir_playbook, playbook,
|
||||
args, main=True)
|
||||
if jobdir_playbook.path is not None:
|
||||
self.jobdir.playbook = jobdir_playbook
|
||||
break
|
||||
if self.jobdir.playbook is None:
|
||||
raise Exception("No valid playbook found")
|
||||
|
||||
for playbook in args['post_playbooks']:
|
||||
jobdir_playbook = self.jobdir.addPostPlaybook()
|
||||
self.preparePlaybookRepo(jobdir_playbook, playbook, args)
|
||||
self.preparePlaybookRepo(jobdir_playbook, playbook,
|
||||
args, main=False)
|
||||
|
||||
def preparePlaybookRepo(self, jobdir_playbook, playbook, args):
|
||||
def preparePlaybookRepo(self, jobdir_playbook, playbook, args, main):
|
||||
self.log.debug("Prepare playbook repo for %s" % (playbook,))
|
||||
# Check out the playbook repo if needed and set the path to
|
||||
# the playbook that should be run.
|
||||
jobdir_playbook.secure = playbook['secure']
|
||||
@ -602,7 +621,7 @@ class AnsibleJob(object):
|
||||
path = os.path.join(self.jobdir.git_root,
|
||||
project.name,
|
||||
playbook['path'])
|
||||
jobdir_playbook.path = self.findPlaybook(path)
|
||||
jobdir_playbook.path = self.findPlaybook(path, main)
|
||||
return
|
||||
# The playbook repo is either a config repo, or it isn't in
|
||||
# the stack of changes we are testing, so check out the branch
|
||||
@ -614,7 +633,7 @@ class AnsibleJob(object):
|
||||
path = os.path.join(jobdir_playbook.root,
|
||||
project.name,
|
||||
playbook['path'])
|
||||
jobdir_playbook.path = self.findPlaybook(path)
|
||||
jobdir_playbook.path = self.findPlaybook(path, main)
|
||||
|
||||
def prepareAnsibleFiles(self, args):
|
||||
with open(self.jobdir.inventory, 'w') as inventory:
|
||||
|
195
zuul/model.py
195
zuul/model.py
@ -531,6 +531,12 @@ class SourceContext(object):
|
||||
self.branch,
|
||||
self.secure)
|
||||
|
||||
def __deepcopy__(self, memo):
|
||||
return self.copy()
|
||||
|
||||
def copy(self):
|
||||
return self.__class__(self.project, self.branch, self.secure)
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self.__eq__(other)
|
||||
|
||||
@ -579,20 +585,20 @@ class PlaybookContext(object):
|
||||
|
||||
class Job(object):
|
||||
|
||||
"""A Job represents the defintion of actions to perform."""
|
||||
"""A Job represents the defintion of actions to perform.
|
||||
|
||||
NB: Do not modify attributes of this class, set them directly
|
||||
(e.g., "job.run = ..." rather than "job.run.append(...)").
|
||||
"""
|
||||
|
||||
def __init__(self, name):
|
||||
self.attributes = dict(
|
||||
timeout=None,
|
||||
# variables={},
|
||||
nodeset=NodeSet(),
|
||||
auth={},
|
||||
workspace=None,
|
||||
pre_run=[],
|
||||
post_run=[],
|
||||
run=None,
|
||||
voting=None,
|
||||
hold_following_changes=None,
|
||||
# These attributes may override even the final form of a job
|
||||
# in the context of a project-pipeline. They can not affect
|
||||
# the execution of the job, but only whether the job is run
|
||||
# and how it is reported.
|
||||
self.context_attributes = dict(
|
||||
voting=True,
|
||||
hold_following_changes=False,
|
||||
failure_message=None,
|
||||
success_message=None,
|
||||
failure_url=None,
|
||||
@ -602,16 +608,44 @@ class Job(object):
|
||||
branch_matcher=None,
|
||||
file_matcher=None,
|
||||
irrelevant_file_matcher=None, # skip-if
|
||||
tags=set(),
|
||||
mutex=None,
|
||||
attempts=3,
|
||||
source_context=None,
|
||||
inheritance_path=[],
|
||||
tags=frozenset(),
|
||||
)
|
||||
|
||||
# These attributes affect how the job is actually run and more
|
||||
# care must be taken when overriding them. If a job is
|
||||
# declared "final", these may not be overriden in a
|
||||
# project-pipeline.
|
||||
self.execution_attributes = dict(
|
||||
timeout=None,
|
||||
# variables={},
|
||||
nodeset=NodeSet(),
|
||||
auth={},
|
||||
workspace=None,
|
||||
pre_run=(),
|
||||
post_run=(),
|
||||
run=(),
|
||||
implied_run=(),
|
||||
mutex=None,
|
||||
attempts=3,
|
||||
final=False,
|
||||
)
|
||||
|
||||
# These are generally internal attributes which are not
|
||||
# accessible via configuration.
|
||||
self.other_attributes = dict(
|
||||
name=None,
|
||||
source_context=None,
|
||||
inheritance_path=(),
|
||||
)
|
||||
|
||||
self.inheritable_attributes = {}
|
||||
self.inheritable_attributes.update(self.context_attributes)
|
||||
self.inheritable_attributes.update(self.execution_attributes)
|
||||
self.attributes = {}
|
||||
self.attributes.update(self.inheritable_attributes)
|
||||
self.attributes.update(self.other_attributes)
|
||||
|
||||
self.name = name
|
||||
for k, v in self.attributes.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self.__eq__(other)
|
||||
@ -637,24 +671,82 @@ class Job(object):
|
||||
self.branch_matcher,
|
||||
self.source_context)
|
||||
|
||||
def inheritFrom(self, other, comment='unknown'):
|
||||
def __getattr__(self, name):
|
||||
v = self.__dict__.get(name)
|
||||
if v is None:
|
||||
return copy.deepcopy(self.attributes[name])
|
||||
return v
|
||||
|
||||
def _get(self, name):
|
||||
return self.__dict__.get(name)
|
||||
|
||||
def setRun(self):
|
||||
if not self.run:
|
||||
self.run = self.implied_run
|
||||
|
||||
def inheritFrom(self, other):
|
||||
"""Copy the inheritable attributes which have been set on the other
|
||||
job to this job."""
|
||||
if not isinstance(other, Job):
|
||||
raise Exception("Job unable to inherit from %s" % (other,))
|
||||
|
||||
do_not_inherit = set()
|
||||
if other.auth and not other.auth.get('inherit'):
|
||||
do_not_inherit.add('auth')
|
||||
|
||||
# copy all attributes
|
||||
for k in self.inheritable_attributes:
|
||||
if (other._get(k) is not None and k not in do_not_inherit):
|
||||
setattr(self, k, copy.deepcopy(getattr(other, k)))
|
||||
|
||||
msg = 'inherit from %s' % (repr(other),)
|
||||
self.inheritance_path = other.inheritance_path + (msg,)
|
||||
|
||||
def copy(self):
|
||||
job = Job(self.name)
|
||||
for k in self.attributes:
|
||||
if self._get(k) is not None:
|
||||
setattr(job, k, copy.deepcopy(self._get(k)))
|
||||
return job
|
||||
|
||||
def applyVariant(self, other):
|
||||
"""Copy the attributes which have been set on the other job to this
|
||||
job."""
|
||||
|
||||
if not isinstance(other, Job):
|
||||
raise Exception("Job unable to inherit from %s" % (other,))
|
||||
self.inheritance_path.extend(other.inheritance_path)
|
||||
self.inheritance_path.append('%s %s' % (repr(other), comment))
|
||||
for k, v in self.attributes.items():
|
||||
if (getattr(other, k) != v and k not in
|
||||
set(['auth', 'pre_run', 'post_run', 'inheritance_path'])):
|
||||
setattr(self, k, getattr(other, k))
|
||||
# Inherit auth only if explicitly allowed
|
||||
if other.auth and 'inherit' in other.auth and other.auth['inherit']:
|
||||
setattr(self, 'auth', getattr(other, 'auth'))
|
||||
# Pre and post run are lists; make a copy
|
||||
self.pre_run = other.pre_run + self.pre_run
|
||||
self.post_run = self.post_run + other.post_run
|
||||
|
||||
for k in self.execution_attributes:
|
||||
if (other._get(k) is not None and
|
||||
k not in set(['final'])):
|
||||
if self.final:
|
||||
raise Exception("Unable to modify final job %s attribute "
|
||||
"%s=%s with variant %s" % (
|
||||
repr(self), k, other._get(k),
|
||||
repr(other)))
|
||||
if k not in set(['pre_run', 'post_run']):
|
||||
setattr(self, k, copy.deepcopy(other._get(k)))
|
||||
|
||||
# Don't set final above so that we don't trip an error halfway
|
||||
# through assignment.
|
||||
if other.final != self.attributes['final']:
|
||||
self.final = other.final
|
||||
|
||||
if other._get('pre_run') is not None:
|
||||
self.pre_run = self.pre_run + other.pre_run
|
||||
if other._get('post_run') is not None:
|
||||
self.post_run = other.post_run + self.post_run
|
||||
|
||||
for k in self.context_attributes:
|
||||
if (other._get(k) is not None and
|
||||
k not in set(['tags'])):
|
||||
setattr(self, k, copy.deepcopy(other._get(k)))
|
||||
|
||||
if other._get('tags') is not None:
|
||||
self.tags = self.tags.union(other.tags)
|
||||
|
||||
msg = 'apply variant %s' % (repr(other),)
|
||||
self.inheritance_path = self.inheritance_path + (msg,)
|
||||
|
||||
def changeMatches(self, change):
|
||||
if self.branch_matcher and not self.branch_matcher.matches(change):
|
||||
@ -710,16 +802,18 @@ class JobTree(object):
|
||||
return ret
|
||||
return None
|
||||
|
||||
def inheritFrom(self, other, comment='unknown'):
|
||||
def inheritFrom(self, other):
|
||||
if other.job:
|
||||
self.job = Job(other.job.name)
|
||||
self.job.inheritFrom(other.job, comment)
|
||||
if not self.job:
|
||||
self.job = other.job.copy()
|
||||
else:
|
||||
self.job.applyVariant(other.job)
|
||||
for other_tree in other.job_trees:
|
||||
this_tree = self.getJobTreeForJob(other_tree.job)
|
||||
if not this_tree:
|
||||
this_tree = JobTree(None)
|
||||
self.job_trees.append(this_tree)
|
||||
this_tree.inheritFrom(other_tree, comment)
|
||||
this_tree.inheritFrom(other_tree)
|
||||
|
||||
|
||||
class Build(object):
|
||||
@ -1984,25 +2078,28 @@ class Layout(object):
|
||||
job = tree.job
|
||||
if not job.changeMatches(change):
|
||||
continue
|
||||
frozen_job = Job(job.name)
|
||||
frozen_tree = JobTree(frozen_job)
|
||||
inherited = set()
|
||||
frozen_job = None
|
||||
matched = False
|
||||
for variant in self.getJobs(job.name):
|
||||
if variant.changeMatches(change):
|
||||
if variant not in inherited:
|
||||
frozen_job.inheritFrom(variant,
|
||||
'variant while freezing')
|
||||
inherited.add(variant)
|
||||
if not inherited:
|
||||
if frozen_job is None:
|
||||
frozen_job = variant.copy()
|
||||
frozen_job.setRun()
|
||||
else:
|
||||
frozen_job.applyVariant(variant)
|
||||
matched = True
|
||||
if not matched:
|
||||
# A change must match at least one defined job variant
|
||||
# (that is to say that it must match more than just
|
||||
# the job that is defined in the tree).
|
||||
continue
|
||||
if job not in inherited:
|
||||
# Only update from the job in the tree if it is
|
||||
# unique, otherwise we might unset an attribute we
|
||||
# have overloaded.
|
||||
frozen_job.inheritFrom(job, 'tree job while freezing')
|
||||
# If the job does not allow auth inheritance, do not allow
|
||||
# the project-pipeline variant to update its execution
|
||||
# attributes.
|
||||
if frozen_job.auth and not frozen_job.auth.get('inherit'):
|
||||
frozen_job.final = True
|
||||
frozen_job.applyVariant(job)
|
||||
frozen_tree = JobTree(frozen_job)
|
||||
parent.job_trees.append(frozen_tree)
|
||||
self._createJobTree(change, tree.job_trees, frozen_tree)
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user