Permit config shadowing
To support the idea that diverse zuul installations can share common job definitions in a 'standard library' project, but still be able to override some job definitions with their own local versions if needed, add a tenant config option to permit a repo to shadow another one. Place the local project first in configuration, then on the remote project, indicate that it shadows the local one. Then, any definitions in the remote repository which conflict with the local will be ignored. Change-Id: Ia715c5fa45141eacbb11449404ee3a3ec948d27f
This commit is contained in:
parent
08d9b7801d
commit
6459db123a
|
@ -2139,6 +2139,8 @@ class ZuulTestCase(BaseTestCase):
|
|||
# Make sure we set up an RSA key for the project so that we
|
||||
# don't spend time generating one:
|
||||
|
||||
if isinstance(project, dict):
|
||||
project = list(project.keys())[0]
|
||||
key_root = os.path.join(self.state_root, 'keys')
|
||||
if not os.path.isdir(key_root):
|
||||
os.mkdir(key_root, 0o700)
|
||||
|
|
|
@ -0,0 +1,2 @@
|
|||
- hosts: all
|
||||
tasks: []
|
|
@ -0,0 +1,2 @@
|
|||
- hosts: all
|
||||
tasks: []
|
|
@ -0,0 +1,25 @@
|
|||
- pipeline:
|
||||
name: check
|
||||
manager: independent
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: patchset-created
|
||||
success:
|
||||
gerrit:
|
||||
verified: 1
|
||||
failure:
|
||||
gerrit:
|
||||
verified: -1
|
||||
|
||||
- job:
|
||||
name: base
|
||||
|
||||
- job:
|
||||
name: test2
|
||||
|
||||
- project:
|
||||
name: org/project
|
||||
check:
|
||||
jobs:
|
||||
- test1
|
||||
- test2
|
|
@ -0,0 +1 @@
|
|||
test
|
|
@ -0,0 +1,10 @@
|
|||
- job:
|
||||
name: base
|
||||
|
||||
- job:
|
||||
name: test1
|
||||
parent: base
|
||||
|
||||
- job:
|
||||
name: test2
|
||||
parent: base
|
|
@ -0,0 +1 @@
|
|||
test
|
|
@ -0,0 +1,2 @@
|
|||
- hosts: all
|
||||
tasks: []
|
|
@ -0,0 +1,2 @@
|
|||
- hosts: all
|
||||
tasks: []
|
|
@ -0,0 +1,2 @@
|
|||
- hosts: all
|
||||
tasks: []
|
|
@ -0,0 +1,10 @@
|
|||
- tenant:
|
||||
name: tenant-one
|
||||
source:
|
||||
gerrit:
|
||||
config-projects:
|
||||
- local-config
|
||||
untrusted-projects:
|
||||
- stdlib:
|
||||
shadow: local-config
|
||||
- org/project
|
|
@ -40,7 +40,7 @@ class TestJob(BaseTestCase):
|
|||
self.source = Dummy(canonical_hostname='git.example.com',
|
||||
connection=self.connection)
|
||||
self.tenant = model.Tenant('tenant')
|
||||
self.layout = model.Layout()
|
||||
self.layout = model.Layout(self.tenant)
|
||||
self.project = model.Project('project', self.source)
|
||||
self.tpc = model.TenantProjectConfig(self.project)
|
||||
self.tenant.addUntrustedProject(self.tpc)
|
||||
|
@ -59,7 +59,7 @@ class TestJob(BaseTestCase):
|
|||
@property
|
||||
def job(self):
|
||||
tenant = model.Tenant('tenant')
|
||||
layout = model.Layout()
|
||||
layout = model.Layout(tenant)
|
||||
job = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': self.context,
|
||||
'_start_mark': self.start_mark,
|
||||
|
@ -170,7 +170,7 @@ class TestJob(BaseTestCase):
|
|||
def test_job_inheritance_configloader(self):
|
||||
# TODO(jeblair): move this to a configloader test
|
||||
tenant = model.Tenant('tenant')
|
||||
layout = model.Layout()
|
||||
layout = model.Layout(tenant)
|
||||
|
||||
pipeline = model.Pipeline('gate', layout)
|
||||
layout.addPipeline(pipeline)
|
||||
|
@ -333,8 +333,8 @@ class TestJob(BaseTestCase):
|
|||
'playbooks/base'])
|
||||
|
||||
def test_job_auth_inheritance(self):
|
||||
tenant = model.Tenant('tenant')
|
||||
layout = model.Layout()
|
||||
tenant = self.tenant
|
||||
layout = self.layout
|
||||
|
||||
conf = yaml.safe_load('''
|
||||
- secret:
|
||||
|
@ -359,7 +359,7 @@ class TestJob(BaseTestCase):
|
|||
secret = configloader.SecretParser.fromYaml(layout, conf)
|
||||
layout.addSecret(secret)
|
||||
|
||||
base = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
base = configloader.JobParser.fromYaml(self.tenant, self.layout, {
|
||||
'_source_context': self.context,
|
||||
'_start_mark': self.start_mark,
|
||||
'name': 'base',
|
||||
|
@ -443,7 +443,7 @@ class TestJob(BaseTestCase):
|
|||
|
||||
def test_job_inheritance_job_tree(self):
|
||||
tenant = model.Tenant('tenant')
|
||||
layout = model.Layout()
|
||||
layout = model.Layout(tenant)
|
||||
tpc = model.TenantProjectConfig(self.project)
|
||||
tenant.addUntrustedProject(tpc)
|
||||
|
||||
|
@ -520,7 +520,7 @@ class TestJob(BaseTestCase):
|
|||
|
||||
def test_inheritance_keeps_matchers(self):
|
||||
tenant = model.Tenant('tenant')
|
||||
layout = model.Layout()
|
||||
layout = model.Layout(tenant)
|
||||
|
||||
pipeline = model.Pipeline('gate', layout)
|
||||
layout.addPipeline(pipeline)
|
||||
|
@ -571,11 +571,13 @@ class TestJob(BaseTestCase):
|
|||
self.assertEqual([], item.getJobs())
|
||||
|
||||
def test_job_source_project(self):
|
||||
tenant = model.Tenant('tenant')
|
||||
layout = model.Layout()
|
||||
tenant = self.tenant
|
||||
layout = self.layout
|
||||
base_project = model.Project('base_project', self.source)
|
||||
base_context = model.SourceContext(base_project, 'master',
|
||||
'test', True)
|
||||
tpc = model.TenantProjectConfig(base_project)
|
||||
tenant.addUntrustedProject(tpc)
|
||||
|
||||
base = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': base_context,
|
||||
|
@ -587,6 +589,8 @@ class TestJob(BaseTestCase):
|
|||
other_project = model.Project('other_project', self.source)
|
||||
other_context = model.SourceContext(other_project, 'master',
|
||||
'test', True)
|
||||
tpc = model.TenantProjectConfig(other_project)
|
||||
tenant.addUntrustedProject(tpc)
|
||||
base2 = configloader.JobParser.fromYaml(tenant, layout, {
|
||||
'_source_context': other_context,
|
||||
'_start_mark': self.start_mark,
|
||||
|
|
|
@ -617,3 +617,17 @@ class TestRoles(ZuulTestCase):
|
|||
self.assertHistory([
|
||||
dict(name='project-test', result='SUCCESS', changes='1,1 2,1'),
|
||||
])
|
||||
|
||||
|
||||
class TestShadow(ZuulTestCase):
|
||||
tenant_config_file = 'config/shadow/main.yaml'
|
||||
|
||||
def test_shadow(self):
|
||||
# Test that a repo is allowed to shadow another's job definitions.
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
self.assertHistory([
|
||||
dict(name='test1', result='SUCCESS', changes='1,1'),
|
||||
dict(name='test2', result='SUCCESS', changes='1,1'),
|
||||
])
|
||||
|
|
|
@ -887,6 +887,7 @@ class TenantParser(object):
|
|||
project_dict = {str: {
|
||||
'include': to_list(classes),
|
||||
'exclude': to_list(classes),
|
||||
'shadow': to_list(str),
|
||||
}}
|
||||
|
||||
project = vs.Any(str, project_dict)
|
||||
|
@ -940,6 +941,10 @@ class TenantParser(object):
|
|||
tenant.addConfigProject(tpc)
|
||||
for tpc in untrusted_tpcs:
|
||||
tenant.addUntrustedProject(tpc)
|
||||
|
||||
for tpc in config_tpcs + untrusted_tpcs:
|
||||
TenantParser._resolveShadowProjects(tenant, tpc)
|
||||
|
||||
tenant.config_projects_config, tenant.untrusted_projects_config = \
|
||||
TenantParser._loadTenantInRepoLayouts(merger, connections,
|
||||
tenant.config_projects,
|
||||
|
@ -953,6 +958,13 @@ class TenantParser(object):
|
|||
connections)
|
||||
return tenant
|
||||
|
||||
@staticmethod
|
||||
def _resolveShadowProjects(tenant, tpc):
|
||||
shadow_projects = []
|
||||
for sp in tpc.shadow_projects:
|
||||
shadow_projects.append(tenant.getProject(sp)[1])
|
||||
tpc.shadow_projects = frozenset(shadow_projects)
|
||||
|
||||
@staticmethod
|
||||
def _loadProjectKeys(project_key_dir, connection_name, project):
|
||||
project.private_key_file = (
|
||||
|
@ -1008,9 +1020,11 @@ class TenantParser(object):
|
|||
# Return a project object whether conf is a dict or a str
|
||||
project = source.getProject(conf)
|
||||
project_include = current_include
|
||||
shadow_projects = []
|
||||
else:
|
||||
project_name = list(conf.keys())[0]
|
||||
project = source.getProject(project_name)
|
||||
shadow_projects = as_list(conf[project_name].get('shadow', []))
|
||||
|
||||
project_include = frozenset(
|
||||
as_list(conf[project_name].get('include', [])))
|
||||
|
@ -1023,6 +1037,7 @@ class TenantParser(object):
|
|||
|
||||
tenant_project_config = model.TenantProjectConfig(project)
|
||||
tenant_project_config.load_classes = frozenset(project_include)
|
||||
tenant_project_config.shadow_projects = shadow_projects
|
||||
|
||||
return tenant_project_config
|
||||
|
||||
|
@ -1234,7 +1249,11 @@ class TenantParser(object):
|
|||
continue
|
||||
with configuration_exceptions('job', config_job):
|
||||
job = JobParser.fromYaml(tenant, layout, config_job)
|
||||
layout.addJob(job)
|
||||
added = layout.addJob(job)
|
||||
if not added:
|
||||
TenantParser.log.debug(
|
||||
"Skipped adding job %s which shadows an existing job" %
|
||||
(job,))
|
||||
|
||||
if not skip_semaphores:
|
||||
for config_semaphore in data.semaphores:
|
||||
|
@ -1273,13 +1292,11 @@ class TenantParser(object):
|
|||
|
||||
@staticmethod
|
||||
def _parseLayout(base, tenant, data, scheduler, connections):
|
||||
layout = model.Layout()
|
||||
layout = model.Layout(tenant)
|
||||
|
||||
TenantParser._parseLayoutItems(layout, tenant, data,
|
||||
scheduler, connections)
|
||||
|
||||
layout.tenant = tenant
|
||||
|
||||
for pipeline in layout.pipelines.values():
|
||||
pipeline.manager._postConfig(layout)
|
||||
|
||||
|
@ -1372,7 +1389,7 @@ class ConfigLoader(object):
|
|||
for project in tenant.untrusted_projects:
|
||||
self._loadDynamicProjectData(config, project, files, False)
|
||||
|
||||
layout = model.Layout()
|
||||
layout = model.Layout(tenant)
|
||||
# NOTE: the actual pipeline objects (complete with queues and
|
||||
# enqueued items) are copied by reference here. This allows
|
||||
# our shadow dynamic configuration to continue to interact
|
||||
|
|
|
@ -2006,6 +2006,7 @@ class TenantProjectConfig(object):
|
|||
def __init__(self, project):
|
||||
self.project = project
|
||||
self.load_classes = set()
|
||||
self.shadow_projects = set()
|
||||
|
||||
|
||||
class ProjectConfig(object):
|
||||
|
@ -2129,8 +2130,8 @@ class UnparsedTenantConfig(object):
|
|||
class Layout(object):
|
||||
"""Holds all of the Pipelines."""
|
||||
|
||||
def __init__(self):
|
||||
self.tenant = None
|
||||
def __init__(self, tenant):
|
||||
self.tenant = tenant
|
||||
self.project_configs = {}
|
||||
self.project_templates = {}
|
||||
self.pipelines = OrderedDict()
|
||||
|
@ -2159,6 +2160,18 @@ class Layout(object):
|
|||
prior_jobs = [j for j in self.getJobs(job.name) if
|
||||
j.source_context.project !=
|
||||
job.source_context.project]
|
||||
# Unless the repo is permitted to shadow another. If so, and
|
||||
# the job we are adding is from a repo that is permitted to
|
||||
# shadow the one with the older jobs, skip adding this job.
|
||||
job_project = job.source_context.project
|
||||
job_tpc = self.tenant.project_configs[job_project.canonical_name]
|
||||
skip_add = False
|
||||
for prior_job in prior_jobs[:]:
|
||||
prior_project = prior_job.source_context.project
|
||||
if prior_project in job_tpc.shadow_projects:
|
||||
prior_jobs.remove(prior_job)
|
||||
skip_add = True
|
||||
|
||||
if prior_jobs:
|
||||
raise Exception("Job %s in %s is not permitted to shadow "
|
||||
"job %s in %s" % (
|
||||
|
@ -2166,11 +2179,13 @@ class Layout(object):
|
|||
job.source_context.project,
|
||||
prior_jobs[0],
|
||||
prior_jobs[0].source_context.project))
|
||||
|
||||
if skip_add:
|
||||
return False
|
||||
if job.name in self.jobs:
|
||||
self.jobs[job.name].append(job)
|
||||
else:
|
||||
self.jobs[job.name] = [job]
|
||||
return True
|
||||
|
||||
def addNodeSet(self, nodeset):
|
||||
if nodeset.name in self.nodesets:
|
||||
|
|
Loading…
Reference in New Issue