zuul/zuul
Code Issues Proposed changes

Support project configs in multiple locations

Browse Source 
If a project definition appears in more than one file (which will
be necessary if a project wants to add a job to its pipeline config
and even more so if that appears in multiple branches) it should be
merged with the existing project config.

To accomplish that, when building the layout, pass all of the
project definitions to the project parser so that we can re-use
the existing template/project-pipeline-config inheritance mechanism
to merge all of the project-pipeline definitions across all of these
configurations.  Rely on the job implicit branch matching to
automatically resolve a given job appearing in the project-pipeline
on multiple branches.

Also, change the in-repo-config zuulv3 test to not use Ansible since
we now have sufficient ansible test coverage elsewhere and this does
not exercise any ansible features.

Change-Id: I5e4cddbc5d29215e2d9da4749c5ec06738e3b305
This commit is contained in:
James E. Blair 2017-02-19 11:34:27 -08:00
parent 859e5fb1c6
commit ff5557467c
4 changed files with 91 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
tests/unit/test_model.py
View File

@ -197,7 +197,7 @@ class TestJob(BaseTestCase):
})
layout.addJob(python27essex)
project_config = configloader.ProjectParser.fromYaml(tenant, layout, {
project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
'_source_context': context,
'name': 'project',
'gate': {
@ -205,7 +205,7 @@ class TestJob(BaseTestCase):
'python27'
]
}
})
}])
layout.addProjectConfig(project_config, update_pipeline=False)
change = model.Change(project)
@ -406,7 +406,7 @@ class TestJob(BaseTestCase):
})
layout.addJob(python27diablo)
project_config = configloader.ProjectParser.fromYaml(tenant, layout, {
project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
'_source_context': context,
'name': 'project',
'gate': {
@ -414,7 +414,7 @@ class TestJob(BaseTestCase):
{'python27': {'timeout': 70}}
]
}
})
}])
layout.addProjectConfig(project_config, update_pipeline=False)
change = model.Change(project)
@ -471,7 +471,7 @@ class TestJob(BaseTestCase):
})
layout.addJob(python27)
project_config = configloader.ProjectParser.fromYaml(tenant, layout, {
project_config = configloader.ProjectParser.fromYaml(tenant, layout, [{
'_source_context': context,
'name': 'project',
'gate': {
@ -479,7 +479,7 @@ class TestJob(BaseTestCase):
'python27',
]
}
})
}])
layout.addProjectConfig(project_config, update_pipeline=False)
change = model.Change(project)

60
tests/unit/test_v3.py
View File

@ -17,7 +17,7 @@
import os
import textwrap
from tests.base import AnsibleZuulTestCase
from tests.base import AnsibleZuulTestCase, ZuulTestCase
class TestMultipleTenants(AnsibleZuulTestCase):
@ -63,7 +63,7 @@ class TestMultipleTenants(AnsibleZuulTestCase):
"not affect tenant one")
class TestInRepoConfig(AnsibleZuulTestCase):
class TestInRepoConfig(ZuulTestCase):
# A temporary class to hold new tests while others are disabled
tenant_config_file = 'config/in-repo/main.yaml'
@ -129,6 +129,62 @@ class TestInRepoConfig(AnsibleZuulTestCase):
dict(name='project-test2', result='SUCCESS', changes='1,1'),
dict(name='project-test2', result='SUCCESS', changes='2,1')])
def test_in_repo_branch(self):
in_repo_conf = textwrap.dedent(
"""
- job:
name: project-test2
- project:
name: org/project
tenant-one-gate:
jobs:
- project-test2
""")
in_repo_playbook = textwrap.dedent(
"""
- hosts: all
tasks: []
""")
file_dict = {'.zuul.yaml': in_repo_conf,
'playbooks/project-test2.yaml': in_repo_playbook}
self.create_branch('org/project', 'stable')
A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A',
files=file_dict)
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2,
"A should report start and success")
self.assertIn('tenant-one-gate', A.messages[1],
"A should transit tenant-one gate")
self.assertHistory([
dict(name='project-test2', result='SUCCESS', changes='1,1')])
self.fake_gerrit.addEvent(A.getChangeMergedEvent())
# The config change should not affect master.
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
B.addApproval('code-review', 2)
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.assertHistory([
dict(name='project-test2', result='SUCCESS', changes='1,1'),
dict(name='project-test1', result='SUCCESS', changes='2,1')])
# The config change should be live for further changes on
# stable.
C = self.fake_gerrit.addFakeChange('org/project', 'stable', 'C')
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
self.assertHistory([
dict(name='project-test2', result='SUCCESS', changes='1,1'),
dict(name='project-test1', result='SUCCESS', changes='2,1'),
dict(name='project-test2', result='SUCCESS', changes='3,1')])
class TestAnsible(AnsibleZuulTestCase):
# A temporary class to hold new tests while others are disabled

43
zuul/configloader.py
View File

@ -327,25 +327,29 @@ class ProjectParser(object):
for p in layout.pipelines.values():
project[p.name] = {'queue': str,
'jobs': [vs.Any(str, dict)]}
return vs.Schema(project)
return vs.Schema([project])
@staticmethod
def fromYaml(tenant, layout, conf):
# TODOv3(jeblair): This may need some branch-specific
# configuration for in-repo configs.
ProjectParser.getSchema(layout)(conf)
# Make a copy since we modify this later via pop
conf = copy.deepcopy(conf)
conf_templates = conf.pop('templates', [])
# The way we construct a project definition is by parsing the
# definition as a template, then applying all of the
# templates, including the newly parsed one, in order.
project_template = ProjectTemplateParser.fromYaml(tenant, layout, conf)
configs = [layout.project_templates[name] for name in conf_templates]
configs.append(project_template)
project = model.ProjectConfig(conf['name'])
mode = conf.get('merge-mode', 'merge-resolve')
def fromYaml(tenant, layout, conf_list):
ProjectParser.getSchema(layout)(conf_list)
project = model.ProjectConfig(conf_list[0]['name'])
mode = conf_list[0].get('merge-mode', 'merge-resolve')
project.merge_mode = model.MERGER_MAP[mode]
# TODOv3(jeblair): deal with merge mode setting on multi branches
configs = []
for conf in conf_list:
# Make a copy since we modify this later via pop
conf = copy.deepcopy(conf)
conf_templates = conf.pop('templates', [])
# The way we construct a project definition is by parsing the
# definition as a template, then applying all of the
# templates, including the newly parsed one, in order.
project_template = ProjectTemplateParser.fromYaml(
tenant, layout, conf)
configs.extend([layout.project_templates[name]
for name in conf_templates])
configs.append(project_template)
for pipeline in layout.pipelines.values():
project_pipeline = model.ProjectPipelineConfig()
project_pipeline.job_tree = model.JobTree(None)
@ -733,7 +737,7 @@ class TenantParser(object):
layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
tenant, layout, config_template))
for config_project in data.projects:
for config_project in data.projects.values():
layout.addProjectConfig(ProjectParser.fromYaml(
tenant, layout, config_project))
@ -790,7 +794,6 @@ class ConfigLoader(object):
def createDynamicLayout(self, tenant, files):
config = tenant.config_repos_config.copy()
for source, project in tenant.project_repos:
# TODOv3(jeblair): config should be branch specific
for branch in source.getProjectBranches(project):
data = files.getFile(project.name, branch, '.zuul.yaml')
if data:
@ -803,7 +806,6 @@ class ConfigLoader(object):
if not incdata:
continue
config.extend(incdata)
layout = model.Layout()
# TODOv3(jeblair): copying the pipelines could be dangerous/confusing.
layout.pipelines = tenant.layout.pipelines
@ -815,8 +817,7 @@ class ConfigLoader(object):
layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
tenant, layout, config_template))
for config_project in config.projects:
for config_project in config.projects.values():
layout.addProjectConfig(ProjectParser.fromYaml(
tenant, layout, config_project), update_pipeline=False)
return layout

8
zuul/model.py
View File

@ -2023,7 +2023,7 @@ class UnparsedTenantConfig(object):
self.pipelines = []
self.jobs = []
self.project_templates = []
self.projects = []
self.projects = {}
self.nodesets = []
def copy(self):
@ -2040,7 +2040,8 @@ class UnparsedTenantConfig(object):
self.pipelines.extend(conf.pipelines)
self.jobs.extend(conf.jobs)
self.project_templates.extend(conf.project_templates)
self.projects.extend(conf.projects)
for k, v in conf.projects.items():
self.projects.setdefault(k, []).extend(v)
self.nodesets.extend(conf.nodesets)
return
@ -2066,7 +2067,8 @@ class UnparsedTenantConfig(object):
if key in ['project', 'project-template', 'job']:
value['_source_context'] = source_context
if key == 'project':
self.projects.append(value)
name = value['name']
self.projects.setdefault(name, []).append(value)
elif key == 'job':
self.jobs.append(value)
elif key == 'project-template':