zuul/zuul
Code Issues Proposed changes

Late bind projects

Browse Source 
Store each independent project stanza as its own config object,
instead of collecting them all together and synthesizing a single
object from all of them.

This will later allow us to re-use unchanged project stanzas
across dynamic configuration changes.

This requires changes to the way project-pipelines are created.
Rather than creating them during configuration loading, they are
now created from the project and project-template config objects
when the job graph is frozen.

We used to create a fixed job list for every project-pipeline,
and in that job list, we would add implied branch matchers to
the project-pipeline job variants based on where the project
stanza was located.  This means that if a 'project:' stanza
within a 'stable' branch listed a certain job, that project-pipeline
job variant would match the 'stable' branch because of the
implied branch matcher added to the project-pipeline job variant.
The same applies to template *invocations* -- that is, where a
project stanza has a 'templates:' attribute.  The templates
themselves didn't have implied branch matchers.

All of that together means that now that we are keeping all of the
individual project stanzas around (instead of combining them at
configuration time) we can move the implied branch matchers from
the project-pipeline job variants to the project stanzas themselves.
Then, when we later dynamically construct the project config based
on the enqueued item, we can just use or ignore entire project
stanzas based on their implied branch matchers.  We no longer need
to modify the project-pipeline job variants.

Change-Id: Ia984ae5c1a4825528a9fa3ea6eb254b75af9b4dd
This commit is contained in:
James E. Blair
2018-03-15 14:41:04 -07:00
parent 90fdac672e
commit c7904bc0b5
10 changed files with 314 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/source/user/config.rst
+29 -1
View File
@@ -1150,12 +1150,34 @@ pipeline.
jobs specified in project-pipeline definitions on the project
itself.
.. attr:: default-branch
:default: master
The name of a branch that Zuul should check out in jobs if no
better match is found. Typically Zuul will check out the branch
which matches the change under test, or if a job has specified
an :attr:`job.override-checkout`, it will check that out.
However, if there is no matching or override branch, then Zuul
will checkout the default branch.
Each project may only have one ``default-branch`` therefore Zuul
will use the first value that it encounters for a given project
(regardless of in which branch the definition appears). It may
not appear in a :ref:`project-template` definition.
.. attr:: merge-mode
:default: merge-resolve
The merge mode which is used by Git for this project. Be sure
this matches what the remote system which performs merges (i.e.,
Gerrit or GitHub). Must be one of the following values:
Gerrit or GitHub).
Each project may only have one ``merge-mode`` therefore Zuul
will use the first value that it encounters for a given project
(regardless of in which branch the definition appears). It may
not appear in a :ref:`project-template` definition.
It must be one of the following values:
.. value:: merge
@@ -1200,6 +1222,12 @@ pipeline.
changes which break the others. This is a free-form string;
just set the same value for each group of projects.
Each pipeline for a project can only belong to one queue,
therefore Zuul will use the first value that it encounters.
It need not appear in the first instance of a :attr:`project`
stanza; it may appear in secondary instances or even in a
:ref:`project-template` definition.
.. attr:: debug
If this is set to `true`, Zuul will include debugging
releasenotes/notes/project-config-e906138042e386f7.yaml
+9
View File
@@ -0,0 +1,9 @@
---
features:
- |
The :attr:`project.default-branch` option is now documented. It has been
supported since version 3.0.0, but was omitted from the documentation.
deprecations:
- |
The ``merge-mode`` and ``default-branch`` attributes may no longer appear
in a :ref:`project-template` stanza.
tests/unit/test_configloader.py
+22 -22
View File
@@ -56,15 +56,15 @@ class TestTenantSimple(TenantParserTestCase):
project1_config = tenant.layout.project_configs.get(
'review.example.com/org/project1')
self.assertTrue('common-config-job' in
project1_config.pipelines['check'].job_list.jobs)
project1_config[0].pipelines['check'].job_list.jobs)
self.assertTrue('project1-job' in
project1_config.pipelines['check'].job_list.jobs)
project1_config[1].pipelines['check'].job_list.jobs)
project2_config = tenant.layout.project_configs.get(
'review.example.com/org/project2')
self.assertTrue('common-config-job' in
project2_config.pipelines['check'].job_list.jobs)
project2_config[0].pipelines['check'].job_list.jobs)
self.assertTrue('project2-job' in
project2_config.pipelines['check'].job_list.jobs)
project2_config[1].pipelines['check'].job_list.jobs)
def test_variant_description(self):
tenant = self.sched.abide.tenants.get('tenant-one')
@@ -98,15 +98,15 @@ class TestTenantOverride(TenantParserTestCase):
project1_config = tenant.layout.project_configs.get(
'review.example.com/org/project1')
self.assertTrue('common-config-job' in
project1_config.pipelines['check'].job_list.jobs)
project1_config[0].pipelines['check'].job_list.jobs)
self.assertFalse('project1-job' in
project1_config.pipelines['check'].job_list.jobs)
project1_config[0].pipelines['check'].job_list.jobs)
project2_config = tenant.layout.project_configs.get(
'review.example.com/org/project2')
self.assertTrue('common-config-job' in
project2_config.pipelines['check'].job_list.jobs)
project2_config[0].pipelines['check'].job_list.jobs)
self.assertFalse('project2-job' in
project2_config.pipelines['check'].job_list.jobs)
project2_config[0].pipelines['check'].job_list.jobs)
class TestTenantGroups(TenantParserTestCase):
@@ -135,15 +135,15 @@ class TestTenantGroups(TenantParserTestCase):
project1_config = tenant.layout.project_configs.get(
'review.example.com/org/project1')
self.assertTrue('common-config-job' in
project1_config.pipelines['check'].job_list.jobs)
project1_config[0].pipelines['check'].job_list.jobs)
self.assertFalse('project1-job' in
project1_config.pipelines['check'].job_list.jobs)
project1_config[0].pipelines['check'].job_list.jobs)
project2_config = tenant.layout.project_configs.get(
'review.example.com/org/project2')
self.assertTrue('common-config-job' in
project2_config.pipelines['check'].job_list.jobs)
project2_config[0].pipelines['check'].job_list.jobs)
self.assertFalse('project2-job' in
project2_config.pipelines['check'].job_list.jobs)
project2_config[0].pipelines['check'].job_list.jobs)
class TestTenantGroups2(TenantParserTestCase):
@@ -172,15 +172,15 @@ class TestTenantGroups2(TenantParserTestCase):
project1_config = tenant.layout.project_configs.get(
'review.example.com/org/project1')
self.assertTrue('common-config-job' in
project1_config.pipelines['check'].job_list.jobs)
project1_config[0].pipelines['check'].job_list.jobs)
self.assertFalse('project1-job' in
project1_config.pipelines['check'].job_list.jobs)
project1_config[0].pipelines['check'].job_list.jobs)
project2_config = tenant.layout.project_configs.get(
'review.example.com/org/project2')
self.assertTrue('common-config-job' in
project2_config.pipelines['check'].job_list.jobs)
project2_config[0].pipelines['check'].job_list.jobs)
self.assertFalse('project2-job' in
project2_config.pipelines['check'].job_list.jobs)
project2_config[0].pipelines['check'].job_list.jobs)
class TestTenantGroups3(TenantParserTestCase):
@@ -208,15 +208,15 @@ class TestTenantGroups3(TenantParserTestCase):
project1_config = tenant.layout.project_configs.get(
'review.example.com/org/project1')
self.assertTrue('common-config-job' in
project1_config.pipelines['check'].job_list.jobs)
project1_config[0].pipelines['check'].job_list.jobs)
self.assertFalse('project1-job' in
project1_config.pipelines['check'].job_list.jobs)
project1_config[0].pipelines['check'].job_list.jobs)
project2_config = tenant.layout.project_configs.get(
'review.example.com/org/project2')
self.assertTrue('common-config-job' in
project2_config.pipelines['check'].job_list.jobs)
project2_config[0].pipelines['check'].job_list.jobs)
self.assertTrue('project2-job' in
project2_config.pipelines['check'].job_list.jobs)
project2_config[1].pipelines['check'].job_list.jobs)
class TestTenantGroups4(TenantParserTestCase):
@@ -289,11 +289,11 @@ class TestSplitConfig(ZuulTestCase):
project_config = tenant.layout.project_configs.get(
'review.example.com/org/project')
self.assertIn('project-test1',
project_config.pipelines['check'].job_list.jobs)
project_config[0].pipelines['check'].job_list.jobs)
project1_config = tenant.layout.project_configs.get(
'review.example.com/org/project1')
self.assertIn('project1-project2-integration',
project1_config.pipelines['check'].job_list.jobs)
project1_config[0].pipelines['check'].job_list.jobs)
def test_dynamic_split_config(self):
in_repo_conf = textwrap.dedent(
tests/unit/test_model.py
+6 -6
View File
@@ -182,7 +182,7 @@ class TestJob(BaseTestCase):
})
self.layout.addJob(python27diablo)
project_config = self.pcontext.project_parser.fromYaml([{
project_config = self.pcontext.project_parser.fromYaml({
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'project',
@@ -192,7 +192,7 @@ class TestJob(BaseTestCase):
'run': 'playbooks/python27.yaml'}}
]
}
}])
})
self.layout.addProjectConfig(project_config)
change = model.Change(self.project)
@@ -247,7 +247,7 @@ class TestJob(BaseTestCase):
})
self.layout.addJob(python27)
project_config = self.pcontext.project_parser.fromYaml([{
project_config = self.pcontext.project_parser.fromYaml({
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'project',
@@ -256,7 +256,7 @@ class TestJob(BaseTestCase):
'python27',
]
}
}])
})
self.layout.addProjectConfig(project_config)
change = model.Change(self.project)
@@ -315,7 +315,7 @@ class TestJob(BaseTestCase):
self.layout.addJob(job)
project_config = self.pcontext.project_parser.fromYaml(
[{
{
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'project',
@@ -324,7 +324,7 @@ class TestJob(BaseTestCase):
'job'
]
}
}]
}
)
self.layout.addProjectConfig(project_config)
tests/unit/test_v3.py
+1
View File
@@ -1920,6 +1920,7 @@ class TestInRepoJoin(ZuulTestCase):
tenant = self.sched.abide.tenants.get('tenant-one')
gate_pipeline = tenant.layout.pipelines['gate']
self.assertEqual(gate_pipeline.queues, [])
in_repo_conf = textwrap.dedent(
"""
zuul/configloader.py
+100 -158
View File
@@ -816,43 +816,43 @@ class ProjectTemplateParser(object):
self.log = logging.getLogger("zuul.ProjectTemplateParser")
self.pcontext = pcontext
self.schema = self.getSchema()
self.not_pipelines = ['name', 'description', 'templates',
'merge-mode', 'default-branch',
'_source_context', '_start_mark']
def getSchema(self):
project_template = {
vs.Required('name'): str,
job = {str: vs.Any(str, JobParser.job_attributes)}
job_list = [vs.Any(str, job)]
pipeline_contents = {
'queue': str,
'debug': bool,
'jobs': job_list
}
project = {
'name': str,
'description': str,
'merge-mode': vs.Any(
'merge', 'merge-resolve',
'cherry-pick'),
str: pipeline_contents,
'_source_context': model.SourceContext,
'_start_mark': ZuulMark,
}
job = {str: vs.Any(str, JobParser.job_attributes)}
job_list = [vs.Any(str, job)]
pipeline_contents = {
'queue': str,
'debug': bool,
'jobs': job_list,
}
for p in self.pcontext.layout.pipelines.values():
project_template[p.name] = pipeline_contents
return vs.Schema(project_template)
return vs.Schema(project)
def fromYaml(self, conf, validate=True):
if validate:
with configuration_exceptions('project-template', conf):
self.schema(conf)
source_context = conf['_source_context']
project_template = model.ProjectConfig(conf['name'], source_context)
project_template = model.ProjectConfig(conf.get('name'),
source_context)
start_mark = conf['_start_mark']
for pipeline in self.pcontext.layout.pipelines.values():
conf_pipeline = conf.get(pipeline.name)
if not conf_pipeline:
for pipeline_name, conf_pipeline in conf.items():
if pipeline_name in self.not_pipelines:
continue
project_pipeline = model.ProjectPipelineConfig()
project_template.pipelines[pipeline.name] = project_pipeline
project_template.pipelines[pipeline_name] = project_pipeline
project_pipeline.queue_name = conf_pipeline.get('queue')
project_pipeline.debug = conf_pipeline.get('debug')
self.parseJobList(
@@ -885,6 +885,15 @@ class ProjectParser(object):
self.schema = self.getSchema()
def getSchema(self):
job = {str: vs.Any(str, JobParser.job_attributes)}
job_list = [vs.Any(str, job)]
pipeline_contents = {
'queue': str,
'debug': bool,
'jobs': job_list
}
project = {
'name': str,
'description': str,
@@ -892,98 +901,53 @@ class ProjectParser(object):
'merge-mode': vs.Any('merge', 'merge-resolve',
'cherry-pick'),
'default-branch': str,
str: pipeline_contents,
'_source_context': model.SourceContext,
'_start_mark': ZuulMark,
}
job = {str: vs.Any(str, JobParser.job_attributes)}
job_list = [vs.Any(str, job)]
pipeline_contents = {
'queue': str,
'debug': bool,
'jobs': job_list
}
for p in self.pcontext.layout.pipelines.values():
project[p.name] = pipeline_contents
return vs.Schema(project)
def fromYaml(self, conf_list):
for conf in conf_list:
with configuration_exceptions('project', conf):
self.schema(conf)
def fromYaml(self, conf):
self.schema(conf)
with configuration_exceptions('project', conf_list[0]):
project_name = conf_list[0]['name']
(trusted, project) = self.pcontext.tenant.getProject(project_name)
if project is None:
raise ProjectNotFoundError(project_name)
project_config = model.ProjectConfig(project.canonical_name)
project_name = conf.get('name')
if not project_name:
# There is no name defined so implicitly add the name
# of the project where it is defined.
project_name = (conf['_source_context'].project.canonical_name)
(trusted, project) = self.pcontext.tenant.getProject(project_name)
if project is None:
raise ProjectNotFoundError(project_name)
configs = []
for conf in conf_list:
implied_branch = None
with configuration_exceptions('project', conf):
if not conf['_source_context'].trusted:
if project != conf['_source_context'].project:
raise ProjectNotPermittedError()
# Parse the project as a template since they're mostly the
# same.
project_config = self.pcontext.project_template_parser.\
fromYaml(conf, validate=False)
project_config.name = project.canonical_name
if not conf['_source_context'].trusted:
if project != conf['_source_context'].project:
raise ProjectNotPermittedError()
# If this project definition is in a place where it
# should get implied branch matchers, set it.
project_config.addImpliedBranchMatcher(
conf['_source_context'].branch)
# Add templates
for name in conf.get('templates', []):
if name not in self.pcontext.layout.project_templates:
raise TemplateNotFoundError(name)
if name not in project_config.templates:
project_config.templates.append(name)
mode = conf.get('merge-mode', 'merge-resolve')
project_config.merge_mode = model.MERGER_MAP[mode]
default_branch = conf.get('default-branch', 'master')
project_config.default_branch = default_branch
conf_templates = conf.get('templates', [])
# The way we construct a project definition is by
# parsing the definition as a template, then applying
# all of the templates, including the newly parsed
# one, in order.
project_template = self.pcontext.project_template_parser.\
fromYaml(conf, validate=False)
# If this project definition is in a place where it
# should get implied branch matchers, set it.
if (not conf['_source_context'].trusted):
implied_branch = conf['_source_context'].branch
for name in conf_templates:
if name not in self.pcontext.layout.project_templates:
raise TemplateNotFoundError(name)
configs.extend([(self.pcontext.layout.project_templates[name],
implied_branch)
for name in conf_templates])
configs.append((project_template, implied_branch))
# Set the following values to the first one that we
# find and ignore subsequent settings.
mode = conf.get('merge-mode')
if mode and project_config.merge_mode is None:
project_config.merge_mode = model.MERGER_MAP[mode]
default_branch = conf.get('default-branch')
if default_branch and project_config.default_branch is None:
project_config.default_branch = default_branch
if project_config.merge_mode is None:
# If merge mode was not specified in any project stanza,
# set it to the default.
project_config.merge_mode = model.MERGER_MAP['merge-resolve']
if project_config.default_branch is None:
project_config.default_branch = 'master'
for pipeline in self.pcontext.layout.pipelines.values():
project_pipeline = model.ProjectPipelineConfig()
queue_name = None
debug = False
# For every template, iterate over the job tree and replace or
# create the jobs in the final definition as needed.
pipeline_defined = False
for (template, implied_branch) in configs:
if pipeline.name in template.pipelines:
pipeline_defined = True
template_pipeline = template.pipelines[pipeline.name]
project_pipeline.job_list.inheritFrom(
template_pipeline.job_list,
implied_branch)
if template_pipeline.queue_name:
queue_name = template_pipeline.queue_name
if template_pipeline.debug is not None:
debug = template_pipeline.debug
if queue_name:
project_pipeline.queue_name = queue_name
if debug:
project_pipeline.debug = True
if pipeline_defined:
project_config.pipelines[pipeline.name] = project_pipeline
return project_config
@@ -1690,62 +1654,40 @@ class TenantParser(object):
pcontext.project_template_parser.fromYaml(
config_template))
flattened_projects = self._flattenProjects(data.projects, tenant)
for config_projects in flattened_projects.values():
# Unlike other config classes, we expect multiple project
# stanzas with the same name, so that a config repo can
# define a project-pipeline and the project itself can
# augment it. To that end, config_project is a list of
# each of the project stanzas. Each one may be (should
# be!) from a different repo, so filter them according to
# the include/exclude rules before parsing them.
filtered_projects = []
for config_project in config_projects:
classes = self._getLoadClasses(tenant, config_project)
if 'project' in classes:
filtered_projects.append(config_project)
if not filtered_projects:
for config_project in data.projects:
classes = self._getLoadClasses(tenant, config_project)
if 'project' not in classes:
continue
layout.addProjectConfig(pcontext.project_parser.fromYaml(
filtered_projects))
# Now that all the project pipelines are loaded, verify
# references to other config objects.
for project_config in layout.project_configs.values():
for project_pipeline_config in project_config.pipelines.values():
for jobs in project_pipeline_config.job_list.jobs.values():
for job in jobs:
with reference_exceptions(
'project or project-template',
job.source_context,
job.start_mark):
# validate that the job exists on its own (an
# additional requirement for project-pipeline
# jobs)
layout.getJob(job.name)
job.validateReferences(layout)
def _flattenProjects(self, projects, tenant):
# Group together all of the project stanzas for each project.
result_projects = {}
for config_project in projects:
with configuration_exceptions('project', config_project):
name = config_project.get('name')
if not name:
# There is no name defined so implicitly add the name
# of the project where it is defined.
name = (config_project['_source_context'].
project.canonical_name)
else:
trusted, project = tenant.getProject(name)
if project is None:
raise ProjectNotFoundError(name)
name = project.canonical_name
config_project['name'] = name
result_projects.setdefault(name, []).append(config_project)
return result_projects
layout.addProjectConfig(
pcontext.project_parser.fromYaml(
config_project))
# Now that all the project pipelines are loaded, fixup and
# verify references to other config objects.
self._validateProjectPipelineConfigs(layout)
def _validateProjectPipelineConfigs(self, layout):
# Validate references to other config objects
ppcs = []
for project_name in layout.project_configs.keys():
for project_config in layout.project_configs[project_name]:
for template_name in project_config.templates:
project_template = layout.getProjectTemplate(template_name)
ppcs.extend(list(project_template.pipelines.values()))
ppcs.extend(list(project_config.pipelines.values()))
for ppc in ppcs:
for jobs in ppc.job_list.jobs.values():
for job in jobs:
with reference_exceptions(
'project',
job.source_context,
job.start_mark):
# validate that the job exists on its own (an
# additional requirement for project-pipeline
# jobs)
layout.getJob(job.name)
job.validateReferences(layout)
def _parseLayout(self, base, tenant, data):
# Don't call this method from dynamic reconfiguration because
zuul/executor/client.py
+8 -8
View File
@@ -201,11 +201,11 @@ class ExecutorClient(object):
for role in d['roles']:
if role['type'] != 'zuul':
continue
project_config = item.layout.project_configs.get(
role['project_canonical_name'], None)
if project_config:
project_metadata = item.layout.getProjectMetadata(
role['project_canonical_name'])
if project_metadata:
role['project_default_branch'] = \
project_config.default_branch
project_metadata.default_branch
else:
role['project_default_branch'] = 'master'
role_trusted, role_project = item.layout.tenant.getProject(
@@ -236,10 +236,10 @@ class ExecutorClient(object):
def make_project_dict(project, override_branch=None,
override_checkout=None):
project_config = item.layout.project_configs.get(
project.canonical_name, None)
if project_config:
project_default_branch = project_config.default_branch
project_metadata = item.layout.getProjectMetadata(
project.canonical_name)
if project_metadata:
project_default_branch = project_metadata.default_branch
else:
project_default_branch = 'master'
connection = project.source.connection
zuul/manager/__init__.py
+2 -53
View File
@@ -58,57 +58,7 @@ class PipelineManager(object):
return "<%s %s>" % (self.__class__.__name__, self.pipeline.name)
def _postConfig(self, layout):
self.log.info("Configured Pipeline Manager %s" % self.pipeline.name)
self.log.info(" Requirements:")
for f in self.ref_filters:
self.log.info(" %s" % f)
self.log.info(" Events:")
for e in self.event_filters:
self.log.info(" %s" % e)
self.log.info(" Projects:")
def log_jobs(job_list):
for job_name, job_variants in job_list.jobs.items():
for variant in job_variants:
# TODOv3(jeblair): represent matchers
efilters = ''
# for b in tree.job._branches:
# efilters += str(b)
# for f in tree.job._files:
# efilters += str(f)
# if tree.job.skip_if_matcher:
# efilters += str(tree.job.skip_if_matcher)
# if efilters:
# efilters = ' ' + efilters
tags = []
if variant.hold_following_changes:
tags.append('[hold]')
if not variant.voting:
tags.append('[nonvoting]')
if variant.semaphore:
tags.append('[semaphore: %s]' % variant.semaphore)
tags = ' '.join(tags)
self.log.info(" %s%s %s" % (repr(variant),
efilters, tags))
for project_name in layout.project_configs.keys():
project_config = layout.project_configs.get(project_name)
if project_config:
project_pipeline_config = project_config.pipelines.get(
self.pipeline.name)
if project_pipeline_config:
self.log.info(" %s" % project_name)
log_jobs(project_pipeline_config.job_list)
self.log.info(" On start:")
self.log.info(" %s" % self.pipeline.start_actions)
self.log.info(" On success:")
self.log.info(" %s" % self.pipeline.success_actions)
self.log.info(" On failure:")
self.log.info(" %s" % self.pipeline.failure_actions)
self.log.info(" On merge-failure:")
self.log.info(" %s" % self.pipeline.merge_failure_actions)
self.log.info(" When disabled:")
self.log.info(" %s" % self.pipeline.disabled_actions)
pass
def getSubmitAllowNeeds(self):
# Get a list of code review labels that are allowed to be
@@ -813,8 +763,7 @@ class PipelineManager(object):
layout = (item.layout or self.pipeline.layout)
project_in_pipeline = True
if not layout.getProjectPipelineConfig(item.change.project,
self.pipeline):
if not layout.getProjectPipelineConfig(item):
self.log.debug("Project %s not in pipeline %s for change %s" % (
item.change.project, self.pipeline, item.change))
project_in_pipeline = False
zuul/manager/dependent.py
+15 -7
View File
@@ -35,16 +35,24 @@ class DependentPipelineManager(PipelineManager):
def buildChangeQueues(self):
self.log.debug("Building shared change queues")
change_queues = {}
project_configs = self.pipeline.layout.project_configs
layout_project_configs = self.pipeline.layout.project_configs
tenant = self.pipeline.layout.tenant
for project_config in project_configs.values():
project_pipeline_config = project_config.pipelines.get(
self.pipeline.name)
if project_pipeline_config is None:
for project_name, project_configs in layout_project_configs.items():
(trusted, project) = tenant.getProject(project_name)
queue_name = None
project_in_pipeline = False
for project_config in project_configs:
project_pipeline_config = project_config.pipelines.get(
self.pipeline.name)
if project_pipeline_config is None:
continue
project_in_pipeline = True
queue_name = project_pipeline_config.queue_name
if queue_name:
break
if not project_in_pipeline:
continue
(trusted, project) = tenant.getProject(project_config.name)
queue_name = project_pipeline_config.queue_name
if queue_name and queue_name in change_queues:
change_queue = change_queues[queue_name]
else:
zuul/model.py
+122 -42
View File
@@ -1255,13 +1255,10 @@ class JobList(object):
else:
self.jobs[job.name] = [job]
def inheritFrom(self, other, implied_branch):
def inheritFrom(self, other):
for jobname, jobs in other.jobs.items():
joblist = self.jobs.setdefault(jobname, [])
for job in jobs:
if implied_branch:
job = job.copy()
job.addImpliedBranchMatcher(implied_branch)
if job not in joblist:
joblist.append(job)
@@ -1581,10 +1578,10 @@ class BuildSet(object):
layout = self.item.pipeline.layout
if layout:
project = self.item.change.project
project_config = layout.project_configs.get(
project_metadata = layout.getProjectMetadata(
project.canonical_name)
if project_config:
return project_config.merge_mode
if project_metadata:
return project_metadata.merge_mode
return MERGER_MERGE_RESOLVE
def getSafeAttributes(self):
@@ -1616,6 +1613,7 @@ class QueueItem(object):
self.active = False # Whether an item is within an active window
self.live = True # Whether an item is intended to be processed at all
self.layout = None
self.project_pipeline_config = None
self.job_graph = None
def __repr__(self):
@@ -1629,6 +1627,7 @@ class QueueItem(object):
def resetAllBuilds(self):
self.current_build_set = BuildSet(self)
self.layout = None
self.project_pipeline_config = None
self.job_graph = None
def addBuild(self, build):
@@ -1641,9 +1640,8 @@ class QueueItem(object):
self.current_build_set.result = result
def debug(self, msg, indent=0):
ppc = self.layout.getProjectPipelineConfig(self.change.project,
self.pipeline)
if not ppc.debug:
if (not self.project_pipeline_config or
not self.project_pipeline_config.debug):
return
if indent:
indent = ' ' * indent
@@ -1654,12 +1652,22 @@ class QueueItem(object):
def freezeJobGraph(self):
"""Find or create actual matching jobs for this item's change and
store the resulting job tree."""
job_graph = self.layout.createJobGraph(self)
for job in job_graph.getJobs():
# Ensure that each jobs's dependencies are fully
# accessible. This will raise an exception if not.
job_graph.getParentJobsRecursively(job.name)
self.job_graph = job_graph
ppc = self.layout.getProjectPipelineConfig(self)
try:
# Conditionally set self.ppc so that the debug method can
# consult it as we resolve the jobs.
self.project_pipeline_config = ppc
job_graph = self.layout.createJobGraph(self, ppc)
for job in job_graph.getJobs():
# Ensure that each jobs's dependencies are fully
# accessible. This will raise an exception if not.
job_graph.getParentJobsRecursively(job.name)
self.job_graph = job_graph
except Exception:
self.project_pipeline_config = None
self.job_graph = None
raise
def hasJobGraph(self):
"""Returns True if the item has a job graph."""
@@ -2394,15 +2402,6 @@ class RefFilter(BaseFilter):
return True
class ProjectPipelineConfig(object):
# Represents a project cofiguration in the context of a pipeline
def __init__(self):
self.job_list = JobList()
self.queue_name = None
self.debug = False
self.merge_mode = None
class TenantProjectConfig(object):
"""A project in the context of a tenant.
@@ -2421,6 +2420,23 @@ class TenantProjectConfig(object):
self.exclude_unprotected_branches = None
class ProjectPipelineConfig(object):
# Represents a project cofiguration in the context of a pipeline
def __init__(self):
self.job_list = JobList()
self.queue_name = None
self.debug = False
def update(self, other):
if not isinstance(other, ProjectPipelineConfig):
raise Exception("Unable to update from %s" % (other,))
if self.queue_name is None:
self.queue_name = other.queue_name
if other.debug:
self.debug = other.debug
self.job_list.inheritFrom(other.job_list)
class ProjectConfig(object):
# Represents a project configuration
def __init__(self, name, source_context=None):
@@ -2428,11 +2444,31 @@ class ProjectConfig(object):
# If this is a template, it will have a source_context, but
# not if it is a project definition.
self.source_context = source_context
self.merge_mode = None
# The default branch for the project (usually master).
self.default_branch = None
self.templates = []
# Pipeline name -> ProjectPipelineConfig
self.pipelines = {}
self.private_key_file = None
self.branch_matcher = None
def addImpliedBranchMatcher(self, branch):
self.branch_matcher = change_matcher.ImpliedBranchMatcher(branch)
def changeMatches(self, change):
if self.branch_matcher and not self.branch_matcher.matches(change):
return False
return True
class ProjectMetadata(object):
"""Information about a Project
A Layout holds one of these for each project it knows about.
Information about the project which is synthesized from multiple
ProjectConfig objects is stored here.
"""
def __init__(self):
self.merge_mode = None
self.default_branch = None
class ConfigItemNotListError(Exception):
@@ -2622,6 +2658,7 @@ class Layout(object):
self.tenant = tenant
self.project_configs = {}
self.project_templates = {}
self.project_metadata = {}
self.pipelines = OrderedDict()
# This is a dictionary of name -> [jobs]. The first element
# of the list is the first job added with that name. It is
@@ -2756,13 +2793,65 @@ class Layout(object):
(project_template.name,))
for pipeline in project_template.pipelines:
template.pipelines[pipeline].job_list.\
inheritFrom(project_template.pipelines[pipeline].job_list,
None)
inheritFrom(project_template.pipelines[pipeline].job_list)
else:
self.project_templates[project_template.name] = project_template
def getProjectTemplate(self, name):
pt = self.project_templates.get(name, None)
if pt is None:
raise Exception("Project template %s not found" % name)
return pt
def addProjectConfig(self, project_config):
self.project_configs[project_config.name] = project_config
if project_config.name in self.project_configs:
self.project_configs[project_config.name].append(project_config)
else:
self.project_configs[project_config.name] = [project_config]
self.project_metadata[project_config.name] = ProjectMetadata()
md = self.project_metadata[project_config.name]
if md.merge_mode is None and project_config.merge_mode is not None:
md.merge_mode = project_config.merge_mode
if (md.default_branch is None and
project_config.default_branch is not None):
md.default_branch = project_config.default_branch
def getProjectConfigs(self, name):
return self.project_configs.get(name, [])
def getProjectMetadata(self, name):
if name in self.project_metadata:
return self.project_metadata[name]
return None
def getProjectPipelineConfig(self, item):
# Create a project-pipeline config for the given item, taking
# its branch (if any) into consideration. If the project does
# not participate in the pipeline at all (in this branch),
# return None.
# A pc for a project can appear only in a config-project
# (unbranched, always applies), or in the project itself (it
# should have an implied branch matcher and it must match the
# item).
ppc = ProjectPipelineConfig()
project_in_pipeline = False
for pc in self.getProjectConfigs(item.change.project.canonical_name):
if not pc.changeMatches(item.change):
continue
for template_name in pc.templates:
template = self.getProjectTemplate(template_name)
template_ppc = template.pipelines.get(item.pipeline.name)
if template_ppc:
project_in_pipeline = True
ppc.update(template_ppc)
project_ppc = pc.pipelines.get(item.pipeline.name)
if project_ppc:
project_in_pipeline = True
ppc.update(project_ppc)
if project_in_pipeline:
return ppc
return None
def _updateOverrideCheckouts(self, override_checkouts, job):
# Update the values in an override_checkouts dict with those
@@ -2934,23 +3023,14 @@ class Layout(object):
frozen_job.name,))
job_graph.addJob(frozen_job)
def createJobGraph(self, item):
def createJobGraph(self, item, ppc):
# NOTE(pabelanger): It is possible for a foreign project not to have a
# configured pipeline, if so return an empty JobGraph.
ret = JobGraph()
ppc = self.getProjectPipelineConfig(item.change.project,
item.pipeline)
if ppc:
self._createJobGraph(item, ppc.job_list, ret)
return ret
def getProjectPipelineConfig(self, project, pipeline):
project_config = self.project_configs.get(
project.canonical_name, None)
if not project_config:
return None
return project_config.pipelines.get(pipeline.name, None)
class Semaphore(object):
def __init__(self, name, max=1):