zuul
/
zuul
Code Issues Proposed changes

Permit config shadowing 

To support the idea that diverse zuul installations can share common
job definitions in a 'standard library' project, but still be able to
override some job definitions with their own local versions if needed,
add a tenant config option to permit a repo to shadow another one.

Place the local project first in configuration, then on the remote project,
indicate that it shadows the local one.  Then, any definitions in the
remote repository which conflict with the local will be ignored.

Change-Id: Ia715c5fa45141eacbb11449404ee3a3ec948d27f
This commit is contained in:
James E. Blair 2017-06-29 14:57:20 -07:00
parent 08d9b7801d
commit 6459db123a
15 changed files with 127 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/base.py
View File

@ -2139,6 +2139,8 @@ class ZuulTestCase(BaseTestCase):
# Make sure we set up an RSA key for the project so that we
# don't spend time generating one:
if isinstance(project, dict):
project = list(project.keys())[0]
key_root = os.path.join(self.state_root, 'keys')
if not os.path.isdir(key_root):
os.mkdir(key_root, 0o700)

2
tests/fixtures/config/shadow/git/local-config/playbooks/base.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

2
tests/fixtures/config/shadow/git/local-config/playbooks/test2.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

25
tests/fixtures/config/shadow/git/local-config/zuul.yaml vendored Normal file
View File

@ -0,0 +1,25 @@
- pipeline:
name: check
manager: independent
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
verified: 1
failure:
gerrit:
verified: -1
- job:
name: base
- job:
name: test2
- project:
name: org/project
check:
jobs:
- test1
- test2

1
tests/fixtures/config/shadow/git/org_project/README vendored Normal file
View File

@ -0,0 +1 @@
test

10
tests/fixtures/config/shadow/git/stdlib/.zuul.yaml vendored Normal file
View File

@ -0,0 +1,10 @@
- job:
name: base
- job:
name: test1
parent: base
- job:
name: test2
parent: base

1
tests/fixtures/config/shadow/git/stdlib/README vendored Normal file
View File

@ -0,0 +1 @@
test

2
tests/fixtures/config/shadow/git/stdlib/playbooks/base.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

2
tests/fixtures/config/shadow/git/stdlib/playbooks/test1.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

2
tests/fixtures/config/shadow/git/stdlib/playbooks/test2.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

10
tests/fixtures/config/shadow/main.yaml vendored Normal file
View File

@ -0,0 +1,10 @@
- tenant:
name: tenant-one
source:
gerrit:
config-projects:
- local-config
untrusted-projects:
- stdlib:
shadow: local-config
- org/project

24
tests/unit/test_model.py
View File

@ -40,7 +40,7 @@ class TestJob(BaseTestCase):
self.source = Dummy(canonical_hostname='git.example.com',
connection=self.connection)
self.tenant = model.Tenant('tenant')
self.layout = model.Layout()
self.layout = model.Layout(self.tenant)
self.project = model.Project('project', self.source)
self.tpc = model.TenantProjectConfig(self.project)
self.tenant.addUntrustedProject(self.tpc)
@ -59,7 +59,7 @@ class TestJob(BaseTestCase):
@property
def job(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
layout = model.Layout(tenant)
job = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': self.context,
'_start_mark': self.start_mark,
@ -170,7 +170,7 @@ class TestJob(BaseTestCase):
def test_job_inheritance_configloader(self):
# TODO(jeblair): move this to a configloader test
tenant = model.Tenant('tenant')
layout = model.Layout()
layout = model.Layout(tenant)
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
@ -333,8 +333,8 @@ class TestJob(BaseTestCase):
'playbooks/base'])
def test_job_auth_inheritance(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
tenant = self.tenant
layout = self.layout
conf = yaml.safe_load('''
- secret:
@ -359,7 +359,7 @@ class TestJob(BaseTestCase):
secret = configloader.SecretParser.fromYaml(layout, conf)
layout.addSecret(secret)
base = configloader.JobParser.fromYaml(tenant, layout, {
base = configloader.JobParser.fromYaml(self.tenant, self.layout, {
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'base',
@ -443,7 +443,7 @@ class TestJob(BaseTestCase):
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
layout = model.Layout(tenant)
tpc = model.TenantProjectConfig(self.project)
tenant.addUntrustedProject(tpc)
@ -520,7 +520,7 @@ class TestJob(BaseTestCase):
def test_inheritance_keeps_matchers(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
layout = model.Layout(tenant)
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
@ -571,11 +571,13 @@ class TestJob(BaseTestCase):
self.assertEqual([], item.getJobs())
def test_job_source_project(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
tenant = self.tenant
layout = self.layout
base_project = model.Project('base_project', self.source)
base_context = model.SourceContext(base_project, 'master',
'test', True)
tpc = model.TenantProjectConfig(base_project)
tenant.addUntrustedProject(tpc)
base = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': base_context,
@ -587,6 +589,8 @@ class TestJob(BaseTestCase):
other_project = model.Project('other_project', self.source)
other_context = model.SourceContext(other_project, 'master',
'test', True)
tpc = model.TenantProjectConfig(other_project)
tenant.addUntrustedProject(tpc)
base2 = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': other_context,
'_start_mark': self.start_mark,

14
tests/unit/test_v3.py
View File

@ -617,3 +617,17 @@ class TestRoles(ZuulTestCase):
self.assertHistory([
dict(name='project-test', result='SUCCESS', changes='1,1 2,1'),
])
class TestShadow(ZuulTestCase):
tenant_config_file = 'config/shadow/main.yaml'
def test_shadow(self):
# Test that a repo is allowed to shadow another's job definitions.
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertHistory([
dict(name='test1', result='SUCCESS', changes='1,1'),
dict(name='test2', result='SUCCESS', changes='1,1'),
])

27
zuul/configloader.py
View File

@ -887,6 +887,7 @@ class TenantParser(object):
project_dict = {str: {
'include': to_list(classes),
'exclude': to_list(classes),
'shadow': to_list(str),
}}
project = vs.Any(str, project_dict)
@ -940,6 +941,10 @@ class TenantParser(object):
tenant.addConfigProject(tpc)
for tpc in untrusted_tpcs:
tenant.addUntrustedProject(tpc)
for tpc in config_tpcs + untrusted_tpcs:
TenantParser._resolveShadowProjects(tenant, tpc)
tenant.config_projects_config, tenant.untrusted_projects_config = \
TenantParser._loadTenantInRepoLayouts(merger, connections,
tenant.config_projects,
@ -953,6 +958,13 @@ class TenantParser(object):
connections)
return tenant
@staticmethod
def _resolveShadowProjects(tenant, tpc):
shadow_projects = []
for sp in tpc.shadow_projects:
shadow_projects.append(tenant.getProject(sp)[1])
tpc.shadow_projects = frozenset(shadow_projects)
@staticmethod
def _loadProjectKeys(project_key_dir, connection_name, project):
project.private_key_file = (
@ -1008,9 +1020,11 @@ class TenantParser(object):
# Return a project object whether conf is a dict or a str
project = source.getProject(conf)
project_include = current_include
shadow_projects = []
else:
project_name = list(conf.keys())[0]
project = source.getProject(project_name)
shadow_projects = as_list(conf[project_name].get('shadow', []))
project_include = frozenset(
as_list(conf[project_name].get('include', [])))
@ -1023,6 +1037,7 @@ class TenantParser(object):
tenant_project_config = model.TenantProjectConfig(project)
tenant_project_config.load_classes = frozenset(project_include)
tenant_project_config.shadow_projects = shadow_projects
return tenant_project_config
@ -1234,7 +1249,11 @@ class TenantParser(object):
continue
with configuration_exceptions('job', config_job):
job = JobParser.fromYaml(tenant, layout, config_job)
layout.addJob(job)
added = layout.addJob(job)
if not added:
TenantParser.log.debug(
"Skipped adding job %s which shadows an existing job" %
(job,))
if not skip_semaphores:
for config_semaphore in data.semaphores:
@ -1273,13 +1292,11 @@ class TenantParser(object):
@staticmethod
def _parseLayout(base, tenant, data, scheduler, connections):
layout = model.Layout()
layout = model.Layout(tenant)
TenantParser._parseLayoutItems(layout, tenant, data,
scheduler, connections)
layout.tenant = tenant
for pipeline in layout.pipelines.values():
pipeline.manager._postConfig(layout)
@ -1372,7 +1389,7 @@ class ConfigLoader(object):
for project in tenant.untrusted_projects:
self._loadDynamicProjectData(config, project, files, False)
layout = model.Layout()
layout = model.Layout(tenant)
# NOTE: the actual pipeline objects (complete with queues and
# enqueued items) are copied by reference here. This allows
# our shadow dynamic configuration to continue to interact

21
zuul/model.py
View File

@ -2006,6 +2006,7 @@ class TenantProjectConfig(object):
def __init__(self, project):
self.project = project
self.load_classes = set()
self.shadow_projects = set()
class ProjectConfig(object):
@ -2129,8 +2130,8 @@ class UnparsedTenantConfig(object):
class Layout(object):
"""Holds all of the Pipelines."""
def __init__(self):
self.tenant = None
def __init__(self, tenant):
self.tenant = tenant
self.project_configs = {}
self.project_templates = {}
self.pipelines = OrderedDict()
@ -2159,6 +2160,18 @@ class Layout(object):
prior_jobs = [j for j in self.getJobs(job.name) if
j.source_context.project !=
job.source_context.project]
# Unless the repo is permitted to shadow another. If so, and
# the job we are adding is from a repo that is permitted to
# shadow the one with the older jobs, skip adding this job.
job_project = job.source_context.project
job_tpc = self.tenant.project_configs[job_project.canonical_name]
skip_add = False
for prior_job in prior_jobs[:]:
prior_project = prior_job.source_context.project
if prior_project in job_tpc.shadow_projects:
prior_jobs.remove(prior_job)
skip_add = True
if prior_jobs:
raise Exception("Job %s in %s is not permitted to shadow "
"job %s in %s" % (
@ -2166,11 +2179,13 @@ class Layout(object):
job.source_context.project,
prior_jobs[0],
prior_jobs[0].source_context.project))
if skip_add:
return False
if job.name in self.jobs:
self.jobs[job.name].append(job)
else:
self.jobs[job.name] = [job]
return True
def addNodeSet(self, nodeset):
if nodeset.name in self.nodesets: