zuul
/
zuul
Code Issues Proposed changes

Merge "Speed configuration building" into feature/zuulv3

This commit is contained in:
Zuul 2017-10-04 22:27:29 +00:00 committed by Gerrit Code Review
parent 51e487e212 af8b208ced
commit e4abb2df60
5 changed files with 246 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
tests/fixtures/layouts/job-vars.yaml vendored Normal file
View File

@ -0,0 +1,75 @@
- pipeline:
name: check
manager: independent
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
Verified: 1
failure:
gerrit:
Verified: -1
- job:
name: base
parent: null
- job:
name: parentjob
parent: base
required-projects:
- org/project0
vars:
override: 0
child1override: 0
parent: 0
- job:
name: child1
parent: parentjob
required-projects:
- org/project1
vars:
override: 1
child1override: 1
child1: 1
- job:
name: child2
parent: parentjob
required-projects:
- org/project2
vars:
override: 2
child2: 2
- job:
name: child3
parent: parentjob
- project:
name: org/project
check:
jobs:
- parentjob
- child1
- child2
- child3:
required-projects:
- org/project3
vars:
override: 3
child3: 3
- project:
name: org/project0
- project:
name: org/project1
- project:
name: org/project2
- project:
name: org/project3

35
tests/unit/test_model.py
View File

@ -246,7 +246,11 @@ class TestJob(BaseTestCase):
})
layout.addJob(python27essex)
project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
project_template_parser = configloader.ProjectTemplateParser(
tenant, layout)
project_parser = configloader.ProjectParser(
tenant, layout, project_template_parser)
project_config = project_parser.fromYaml([{
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'project',
@ -505,6 +509,7 @@ class TestJob(BaseTestCase):
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
layout = model.Layout(tenant)
tpc = model.TenantProjectConfig(self.project)
tenant.addUntrustedProject(tpc)
@ -539,7 +544,11 @@ class TestJob(BaseTestCase):
})
layout.addJob(python27diablo)
project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
project_template_parser = configloader.ProjectTemplateParser(
tenant, layout)
project_parser = configloader.ProjectParser(
tenant, layout, project_template_parser)
project_config = project_parser.fromYaml([{
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'project',
@ -609,7 +618,11 @@ class TestJob(BaseTestCase):
})
layout.addJob(python27)
project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
project_template_parser = configloader.ProjectTemplateParser(
tenant, layout)
project_parser = configloader.ProjectParser(
tenant, layout, project_template_parser)
project_config = project_parser.fromYaml([{
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'project',
@ -682,8 +695,12 @@ class TestJob(BaseTestCase):
context2 = model.SourceContext(project2, 'master',
'test', True)
project2_config = configloader.ProjectParser.fromYaml(
self.tenant, self.layout, [{
project_template_parser = configloader.ProjectTemplateParser(
self.tenant, self.layout)
project_parser = configloader.ProjectParser(
self.tenant, self.layout, project_template_parser)
project2_config = project_parser.fromYaml(
[{
'_source_context': context2,
'_start_mark': self.start_mark,
'name': 'project2',
@ -718,8 +735,12 @@ class TestJob(BaseTestCase):
self.layout.addJob(job)
project_config = configloader.ProjectParser.fromYaml(
self.tenant, self.layout, [{
project_template_parser = configloader.ProjectTemplateParser(
self.tenant, self.layout)
project_parser = configloader.ProjectParser(
self.tenant, self.layout, project_template_parser)
project_config = project_parser.fromYaml(
[{
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'project',

52
tests/unit/test_scheduler.py
View File

@ -2269,6 +2269,58 @@ class TestScheduler(ZuulTestCase):
self.assertEqual(set(['project-test-nomatch-starts-empty',
'project-test-nomatch-starts-full']), run_jobs)
@simple_layout('layouts/job-vars.yaml')
def test_inherited_job_variables(self):
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertHistory([
dict(name='parentjob', result='SUCCESS'),
dict(name='child1', result='SUCCESS'),
dict(name='child2', result='SUCCESS'),
dict(name='child3', result='SUCCESS'),
], ordered=False)
j = self.getJobFromHistory('parentjob')
rp = set([p['name'] for p in j.parameters['projects']])
self.assertEqual(j.parameters['vars']['override'], 0)
self.assertEqual(j.parameters['vars']['child1override'], 0)
self.assertEqual(j.parameters['vars']['parent'], 0)
self.assertFalse('child1' in j.parameters['vars'])
self.assertFalse('child2' in j.parameters['vars'])
self.assertFalse('child3' in j.parameters['vars'])
self.assertEqual(rp, set(['org/project', 'org/project0',
'org/project0']))
j = self.getJobFromHistory('child1')
rp = set([p['name'] for p in j.parameters['projects']])
self.assertEqual(j.parameters['vars']['override'], 1)
self.assertEqual(j.parameters['vars']['child1override'], 1)
self.assertEqual(j.parameters['vars']['parent'], 0)
self.assertEqual(j.parameters['vars']['child1'], 1)
self.assertFalse('child2' in j.parameters['vars'])
self.assertFalse('child3' in j.parameters['vars'])
self.assertEqual(rp, set(['org/project', 'org/project0',
'org/project1']))
j = self.getJobFromHistory('child2')
rp = set([p['name'] for p in j.parameters['projects']])
self.assertEqual(j.parameters['vars']['override'], 2)
self.assertEqual(j.parameters['vars']['child1override'], 0)
self.assertEqual(j.parameters['vars']['parent'], 0)
self.assertFalse('child1' in j.parameters['vars'])
self.assertEqual(j.parameters['vars']['child2'], 2)
self.assertFalse('child3' in j.parameters['vars'])
self.assertEqual(rp, set(['org/project', 'org/project0',
'org/project2']))
j = self.getJobFromHistory('child3')
rp = set([p['name'] for p in j.parameters['projects']])
self.assertEqual(j.parameters['vars']['override'], 3)
self.assertEqual(j.parameters['vars']['child1override'], 0)
self.assertEqual(j.parameters['vars']['parent'], 0)
self.assertFalse('child1' in j.parameters['vars'])
self.assertFalse('child2' in j.parameters['vars'])
self.assertEqual(j.parameters['vars']['child3'], 3)
self.assertEqual(rp, set(['org/project', 'org/project0',
'org/project3']))
def test_queue_names(self):
"Test shared change queue names"
tenant = self.sched.abide.tenants.get('tenant-one')

177
zuul/configloader.py
View File

@ -383,57 +383,55 @@ class SecretParser(object):
class JobParser(object):
ANSIBLE_ROLE_RE = re.compile(r'^(ansible[-_.+]*)*(role[-_.+]*)*')
@staticmethod
def getSchema():
zuul_role = {vs.Required('zuul'): str,
'name': str}
zuul_role = {vs.Required('zuul'): str,
'name': str}
galaxy_role = {vs.Required('galaxy'): str,
'name': str}
galaxy_role = {vs.Required('galaxy'): str,
'name': str}
role = vs.Any(zuul_role, galaxy_role)
role = vs.Any(zuul_role, galaxy_role)
job_project = {vs.Required('name'): str,
'override-branch': str}
job_project = {vs.Required('name'): str,
'override-branch': str}
secret = {vs.Required('name'): str,
vs.Required('secret'): str}
secret = {vs.Required('name'): str,
vs.Required('secret'): str}
job = {vs.Required('name'): str,
'parent': vs.Any(str, None),
'final': bool,
'failure-message': str,
'success-message': str,
'failure-url': str,
'success-url': str,
'hold-following-changes': bool,
'voting': bool,
'semaphore': str,
'tags': to_list(str),
'branches': to_list(str),
'files': to_list(str),
'secrets': to_list(vs.Any(secret, str)),
'irrelevant-files': to_list(str),
# validation happens in NodeSetParser
'nodeset': vs.Any(dict, str),
'timeout': int,
'attempts': int,
'pre-run': to_list(str),
'post-run': to_list(str),
'run': str,
'_source_context': model.SourceContext,
'_start_mark': ZuulMark,
'roles': to_list(role),
'required-projects': to_list(vs.Any(job_project, str)),
'vars': dict,
'dependencies': to_list(str),
'allowed-projects': to_list(str),
'override-branch': str,
'description': str,
'post-review': bool
}
job = {vs.Required('name'): str,
'parent': vs.Any(str, None),
'final': bool,
'failure-message': str,
'success-message': str,
'failure-url': str,
'success-url': str,
'hold-following-changes': bool,
'voting': bool,
'semaphore': str,
'tags': to_list(str),
'branches': to_list(str),
'files': to_list(str),
'secrets': to_list(vs.Any(secret, str)),
'irrelevant-files': to_list(str),
# validation happens in NodeSetParser
'nodeset': vs.Any(dict, str),
'timeout': int,
'attempts': int,
'pre-run': to_list(str),
'post-run': to_list(str),
'run': str,
'_source_context': model.SourceContext,
'_start_mark': ZuulMark,
'roles': to_list(role),
'required-projects': to_list(vs.Any(job_project, str)),
'vars': dict,
'dependencies': to_list(str),
'allowed-projects': to_list(str),
'override-branch': str,
'description': str,
'post-review': bool
}
return vs.Schema(job)
schema = vs.Schema(job)
simple_attributes = [
'final',
@ -478,7 +476,7 @@ class JobParser(object):
@staticmethod
def fromYaml(tenant, layout, conf, project_pipeline=False):
with configuration_exceptions('job', conf):
JobParser.getSchema()(conf)
JobParser.schema(conf)
# NB: The default detection system in the Job class requires
# that we always assign values directly rather than modifying
@ -710,10 +708,13 @@ class JobParser(object):
class ProjectTemplateParser(object):
log = logging.getLogger("zuul.ProjectTemplateParser")
def __init__(self, tenant, layout):
self.log = logging.getLogger("zuul.ProjectTemplateParser")
self.tenant = tenant
self.layout = layout
self.schema = self.getSchema()
@staticmethod
def getSchema(layout):
def getSchema(self):
project_template = {
vs.Required('name'): str,
'description': str,
@ -724,40 +725,31 @@ class ProjectTemplateParser(object):
'_start_mark': ZuulMark,
}
for p in layout.pipelines.values():
for p in self.layout.pipelines.values():
project_template[p.name] = {'queue': str,
'jobs': [vs.Any(str, dict)]}
return vs.Schema(project_template)
@staticmethod
def fromYaml(tenant, layout, conf, template):
if template:
project_or_template = 'project-template'
else:
project_or_template = 'project'
with configuration_exceptions(project_or_template, conf):
ProjectTemplateParser.getSchema(layout)(conf)
# Make a copy since we modify this later via pop
conf = copy.deepcopy(conf)
def fromYaml(self, conf, validate=True):
if validate:
with configuration_exceptions('project-template', conf):
self.schema(conf)
project_template = model.ProjectConfig(conf['name'])
source_context = conf['_source_context']
start_mark = conf['_start_mark']
for pipeline in layout.pipelines.values():
for pipeline in self.layout.pipelines.values():
conf_pipeline = conf.get(pipeline.name)
if not conf_pipeline:
continue
project_pipeline = model.ProjectPipelineConfig()
project_template.pipelines[pipeline.name] = project_pipeline
project_pipeline.queue_name = conf_pipeline.get('queue')
ProjectTemplateParser._parseJobList(
tenant, layout, conf_pipeline.get('jobs', []),
source_context, start_mark, project_pipeline.job_list,
template)
self.parseJobList(
conf_pipeline.get('jobs', []),
source_context, start_mark, project_pipeline.job_list)
return project_template
@staticmethod
def _parseJobList(tenant, layout, conf, source_context,
start_mark, job_list, template):
def parseJobList(self, conf, source_context, start_mark, job_list):
for conf_job in conf:
if isinstance(conf_job, str):
attrs = dict(name=conf_job)
@ -778,17 +770,21 @@ class ProjectTemplateParser(object):
# validate that the job is existing
with configuration_exceptions('project or project-template',
attrs):
layout.getJob(attrs['name'])
self.layout.getJob(attrs['name'])
job_list.addJob(JobParser.fromYaml(tenant, layout, attrs,
project_pipeline=True))
job_list.addJob(JobParser.fromYaml(self.tenant, self.layout,
attrs, project_pipeline=True))
class ProjectParser(object):
log = logging.getLogger("zuul.ProjectParser")
def __init__(self, tenant, layout, project_template_parser):
self.log = logging.getLogger("zuul.ProjectParser")
self.tenant = tenant
self.layout = layout
self.project_template_parser = project_template_parser
self.schema = self.getSchema()
@staticmethod
def getSchema(layout):
def getSchema(self):
project = {
vs.Required('name'): str,
'description': str,
@ -800,20 +796,19 @@ class ProjectParser(object):
'_start_mark': ZuulMark,
}
for p in layout.pipelines.values():
for p in self.layout.pipelines.values():
project[p.name] = {'queue': str,
'jobs': [vs.Any(str, dict)]}
return vs.Schema(project)
@staticmethod
def fromYaml(tenant, layout, conf_list):
def fromYaml(self, conf_list):
for conf in conf_list:
with configuration_exceptions('project', conf):
ProjectParser.getSchema(layout)(conf)
self.schema(conf)
with configuration_exceptions('project', conf_list[0]):
project_name = conf_list[0]['name']
(trusted, project) = tenant.getProject(project_name)
(trusted, project) = self.tenant.getProject(project_name)
if project is None:
raise ProjectNotFoundError(project_name)
project_config = model.ProjectConfig(project.canonical_name)
@ -826,23 +821,21 @@ class ProjectParser(object):
if project != conf['_source_context'].project:
raise ProjectNotPermittedError()
# Make a copy since we modify this later via pop
conf = copy.deepcopy(conf)
conf_templates = conf.pop('templates', [])
conf_templates = conf.get('templates', [])
# The way we construct a project definition is by
# parsing the definition as a template, then applying
# all of the templates, including the newly parsed
# one, in order.
project_template = ProjectTemplateParser.fromYaml(
tenant, layout, conf, template=False)
project_template = self.project_template_parser.fromYaml(
conf, validate=False)
# If this project definition is in a place where it
# should get implied branch matchers, set it.
if (not conf['_source_context'].trusted):
implied_branch = conf['_source_context'].branch
for name in conf_templates:
if name not in layout.project_templates:
if name not in self.layout.project_templates:
raise TemplateNotFoundError(name)
configs.extend([(layout.project_templates[name],
configs.extend([(self.layout.project_templates[name],
implied_branch)
for name in conf_templates])
configs.append((project_template, implied_branch))
@ -860,7 +853,7 @@ class ProjectParser(object):
project_config.merge_mode = model.MERGER_MAP['merge-resolve']
if project_config.default_branch is None:
project_config.default_branch = 'master'
for pipeline in layout.pipelines.values():
for pipeline in self.layout.pipelines.values():
project_pipeline = model.ProjectPipelineConfig()
queue_name = None
# For every template, iterate over the job tree and replace or
@ -1527,13 +1520,15 @@ class TenantParser(object):
continue
layout.addSemaphore(semaphore)
project_template_parser = ProjectTemplateParser(tenant, layout)
for config_template in data.project_templates:
classes = TenantParser._getLoadClasses(tenant, config_template)
if 'project-template' not in classes:
continue
layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
tenant, layout, config_template, template=True))
layout.addProjectTemplate(project_template_parser.fromYaml(
config_template))
project_parser = ProjectParser(tenant, layout, project_template_parser)
for config_projects in data.projects.values():
# Unlike other config classes, we expect multiple project
# stanzas with the same name, so that a config repo can
@ -1551,8 +1546,8 @@ class TenantParser(object):
if not filtered_projects:
continue
layout.addProjectConfig(ProjectParser.fromYaml(
tenant, layout, filtered_projects))
layout.addProjectConfig(project_parser.fromYaml(
filtered_projects))
@staticmethod
def _parseLayout(base, tenant, data, scheduler, connections):

10
zuul/model.py
View File

@ -877,7 +877,7 @@ class Job(object):
def __getattr__(self, name):
v = self.__dict__.get(name)
if v is None:
return copy.deepcopy(self.attributes[name])
return self.attributes[name]
return v
def _get(self, name):
@ -925,13 +925,13 @@ class Job(object):
self.branch_matcher = change_matcher.MatchAny(matchers)
def updateVariables(self, other_vars):
v = self.variables
v = copy.deepcopy(self.variables)
Job._deepUpdate(v, other_vars)
self.variables = v
def updateProjects(self, other_projects):
required_projects = self.required_projects
Job._deepUpdate(required_projects, other_projects)
required_projects = self.required_projects.copy()
required_projects.update(other_projects)
self.required_projects = required_projects
@staticmethod
@ -958,7 +958,7 @@ class Job(object):
# copy all attributes
for k in self.inheritable_attributes:
if (other._get(k) is not None):
setattr(self, k, copy.deepcopy(getattr(other, k)))
setattr(self, k, getattr(other, k))
msg = 'inherit from %s' % (repr(other),)
self.inheritance_path = other.inheritance_path + (msg,)