Permit config shadowing
To support the idea that diverse zuul installations can share common job definitions in a 'standard library' project, but still be able to override some job definitions with their own local versions if needed, add a tenant config option to permit a repo to shadow another one. Place the local project first in configuration, then on the remote project, indicate that it shadows the local one. Then, any definitions in the remote repository which conflict with the local will be ignored. Change-Id: Ia715c5fa45141eacbb11449404ee3a3ec948d27f
This commit is contained in:
parent
08d9b7801d
commit
6459db123a
|
@ -2139,6 +2139,8 @@ class ZuulTestCase(BaseTestCase):
|
||||||
# Make sure we set up an RSA key for the project so that we
|
# Make sure we set up an RSA key for the project so that we
|
||||||
# don't spend time generating one:
|
# don't spend time generating one:
|
||||||
|
|
||||||
|
if isinstance(project, dict):
|
||||||
|
project = list(project.keys())[0]
|
||||||
key_root = os.path.join(self.state_root, 'keys')
|
key_root = os.path.join(self.state_root, 'keys')
|
||||||
if not os.path.isdir(key_root):
|
if not os.path.isdir(key_root):
|
||||||
os.mkdir(key_root, 0o700)
|
os.mkdir(key_root, 0o700)
|
||||||
|
|
|
@ -0,0 +1,2 @@
|
||||||
|
- hosts: all
|
||||||
|
tasks: []
|
|
@ -0,0 +1,2 @@
|
||||||
|
- hosts: all
|
||||||
|
tasks: []
|
|
@ -0,0 +1,25 @@
|
||||||
|
- pipeline:
|
||||||
|
name: check
|
||||||
|
manager: independent
|
||||||
|
trigger:
|
||||||
|
gerrit:
|
||||||
|
- event: patchset-created
|
||||||
|
success:
|
||||||
|
gerrit:
|
||||||
|
verified: 1
|
||||||
|
failure:
|
||||||
|
gerrit:
|
||||||
|
verified: -1
|
||||||
|
|
||||||
|
- job:
|
||||||
|
name: base
|
||||||
|
|
||||||
|
- job:
|
||||||
|
name: test2
|
||||||
|
|
||||||
|
- project:
|
||||||
|
name: org/project
|
||||||
|
check:
|
||||||
|
jobs:
|
||||||
|
- test1
|
||||||
|
- test2
|
|
@ -0,0 +1 @@
|
||||||
|
test
|
|
@ -0,0 +1,10 @@
|
||||||
|
- job:
|
||||||
|
name: base
|
||||||
|
|
||||||
|
- job:
|
||||||
|
name: test1
|
||||||
|
parent: base
|
||||||
|
|
||||||
|
- job:
|
||||||
|
name: test2
|
||||||
|
parent: base
|
|
@ -0,0 +1 @@
|
||||||
|
test
|
|
@ -0,0 +1,2 @@
|
||||||
|
- hosts: all
|
||||||
|
tasks: []
|
|
@ -0,0 +1,2 @@
|
||||||
|
- hosts: all
|
||||||
|
tasks: []
|
|
@ -0,0 +1,2 @@
|
||||||
|
- hosts: all
|
||||||
|
tasks: []
|
|
@ -0,0 +1,10 @@
|
||||||
|
- tenant:
|
||||||
|
name: tenant-one
|
||||||
|
source:
|
||||||
|
gerrit:
|
||||||
|
config-projects:
|
||||||
|
- local-config
|
||||||
|
untrusted-projects:
|
||||||
|
- stdlib:
|
||||||
|
shadow: local-config
|
||||||
|
- org/project
|
|
@ -40,7 +40,7 @@ class TestJob(BaseTestCase):
|
||||||
self.source = Dummy(canonical_hostname='git.example.com',
|
self.source = Dummy(canonical_hostname='git.example.com',
|
||||||
connection=self.connection)
|
connection=self.connection)
|
||||||
self.tenant = model.Tenant('tenant')
|
self.tenant = model.Tenant('tenant')
|
||||||
self.layout = model.Layout()
|
self.layout = model.Layout(self.tenant)
|
||||||
self.project = model.Project('project', self.source)
|
self.project = model.Project('project', self.source)
|
||||||
self.tpc = model.TenantProjectConfig(self.project)
|
self.tpc = model.TenantProjectConfig(self.project)
|
||||||
self.tenant.addUntrustedProject(self.tpc)
|
self.tenant.addUntrustedProject(self.tpc)
|
||||||
|
@ -59,7 +59,7 @@ class TestJob(BaseTestCase):
|
||||||
@property
|
@property
|
||||||
def job(self):
|
def job(self):
|
||||||
tenant = model.Tenant('tenant')
|
tenant = model.Tenant('tenant')
|
||||||
layout = model.Layout()
|
layout = model.Layout(tenant)
|
||||||
job = configloader.JobParser.fromYaml(tenant, layout, {
|
job = configloader.JobParser.fromYaml(tenant, layout, {
|
||||||
'_source_context': self.context,
|
'_source_context': self.context,
|
||||||
'_start_mark': self.start_mark,
|
'_start_mark': self.start_mark,
|
||||||
|
@ -170,7 +170,7 @@ class TestJob(BaseTestCase):
|
||||||
def test_job_inheritance_configloader(self):
|
def test_job_inheritance_configloader(self):
|
||||||
# TODO(jeblair): move this to a configloader test
|
# TODO(jeblair): move this to a configloader test
|
||||||
tenant = model.Tenant('tenant')
|
tenant = model.Tenant('tenant')
|
||||||
layout = model.Layout()
|
layout = model.Layout(tenant)
|
||||||
|
|
||||||
pipeline = model.Pipeline('gate', layout)
|
pipeline = model.Pipeline('gate', layout)
|
||||||
layout.addPipeline(pipeline)
|
layout.addPipeline(pipeline)
|
||||||
|
@ -333,8 +333,8 @@ class TestJob(BaseTestCase):
|
||||||
'playbooks/base'])
|
'playbooks/base'])
|
||||||
|
|
||||||
def test_job_auth_inheritance(self):
|
def test_job_auth_inheritance(self):
|
||||||
tenant = model.Tenant('tenant')
|
tenant = self.tenant
|
||||||
layout = model.Layout()
|
layout = self.layout
|
||||||
|
|
||||||
conf = yaml.safe_load('''
|
conf = yaml.safe_load('''
|
||||||
- secret:
|
- secret:
|
||||||
|
@ -359,7 +359,7 @@ class TestJob(BaseTestCase):
|
||||||
secret = configloader.SecretParser.fromYaml(layout, conf)
|
secret = configloader.SecretParser.fromYaml(layout, conf)
|
||||||
layout.addSecret(secret)
|
layout.addSecret(secret)
|
||||||
|
|
||||||
base = configloader.JobParser.fromYaml(tenant, layout, {
|
base = configloader.JobParser.fromYaml(self.tenant, self.layout, {
|
||||||
'_source_context': self.context,
|
'_source_context': self.context,
|
||||||
'_start_mark': self.start_mark,
|
'_start_mark': self.start_mark,
|
||||||
'name': 'base',
|
'name': 'base',
|
||||||
|
@ -443,7 +443,7 @@ class TestJob(BaseTestCase):
|
||||||
|
|
||||||
def test_job_inheritance_job_tree(self):
|
def test_job_inheritance_job_tree(self):
|
||||||
tenant = model.Tenant('tenant')
|
tenant = model.Tenant('tenant')
|
||||||
layout = model.Layout()
|
layout = model.Layout(tenant)
|
||||||
tpc = model.TenantProjectConfig(self.project)
|
tpc = model.TenantProjectConfig(self.project)
|
||||||
tenant.addUntrustedProject(tpc)
|
tenant.addUntrustedProject(tpc)
|
||||||
|
|
||||||
|
@ -520,7 +520,7 @@ class TestJob(BaseTestCase):
|
||||||
|
|
||||||
def test_inheritance_keeps_matchers(self):
|
def test_inheritance_keeps_matchers(self):
|
||||||
tenant = model.Tenant('tenant')
|
tenant = model.Tenant('tenant')
|
||||||
layout = model.Layout()
|
layout = model.Layout(tenant)
|
||||||
|
|
||||||
pipeline = model.Pipeline('gate', layout)
|
pipeline = model.Pipeline('gate', layout)
|
||||||
layout.addPipeline(pipeline)
|
layout.addPipeline(pipeline)
|
||||||
|
@ -571,11 +571,13 @@ class TestJob(BaseTestCase):
|
||||||
self.assertEqual([], item.getJobs())
|
self.assertEqual([], item.getJobs())
|
||||||
|
|
||||||
def test_job_source_project(self):
|
def test_job_source_project(self):
|
||||||
tenant = model.Tenant('tenant')
|
tenant = self.tenant
|
||||||
layout = model.Layout()
|
layout = self.layout
|
||||||
base_project = model.Project('base_project', self.source)
|
base_project = model.Project('base_project', self.source)
|
||||||
base_context = model.SourceContext(base_project, 'master',
|
base_context = model.SourceContext(base_project, 'master',
|
||||||
'test', True)
|
'test', True)
|
||||||
|
tpc = model.TenantProjectConfig(base_project)
|
||||||
|
tenant.addUntrustedProject(tpc)
|
||||||
|
|
||||||
base = configloader.JobParser.fromYaml(tenant, layout, {
|
base = configloader.JobParser.fromYaml(tenant, layout, {
|
||||||
'_source_context': base_context,
|
'_source_context': base_context,
|
||||||
|
@ -587,6 +589,8 @@ class TestJob(BaseTestCase):
|
||||||
other_project = model.Project('other_project', self.source)
|
other_project = model.Project('other_project', self.source)
|
||||||
other_context = model.SourceContext(other_project, 'master',
|
other_context = model.SourceContext(other_project, 'master',
|
||||||
'test', True)
|
'test', True)
|
||||||
|
tpc = model.TenantProjectConfig(other_project)
|
||||||
|
tenant.addUntrustedProject(tpc)
|
||||||
base2 = configloader.JobParser.fromYaml(tenant, layout, {
|
base2 = configloader.JobParser.fromYaml(tenant, layout, {
|
||||||
'_source_context': other_context,
|
'_source_context': other_context,
|
||||||
'_start_mark': self.start_mark,
|
'_start_mark': self.start_mark,
|
||||||
|
|
|
@ -617,3 +617,17 @@ class TestRoles(ZuulTestCase):
|
||||||
self.assertHistory([
|
self.assertHistory([
|
||||||
dict(name='project-test', result='SUCCESS', changes='1,1 2,1'),
|
dict(name='project-test', result='SUCCESS', changes='1,1 2,1'),
|
||||||
])
|
])
|
||||||
|
|
||||||
|
|
||||||
|
class TestShadow(ZuulTestCase):
|
||||||
|
tenant_config_file = 'config/shadow/main.yaml'
|
||||||
|
|
||||||
|
def test_shadow(self):
|
||||||
|
# Test that a repo is allowed to shadow another's job definitions.
|
||||||
|
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||||
|
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||||
|
self.waitUntilSettled()
|
||||||
|
self.assertHistory([
|
||||||
|
dict(name='test1', result='SUCCESS', changes='1,1'),
|
||||||
|
dict(name='test2', result='SUCCESS', changes='1,1'),
|
||||||
|
])
|
||||||
|
|
|
@ -887,6 +887,7 @@ class TenantParser(object):
|
||||||
project_dict = {str: {
|
project_dict = {str: {
|
||||||
'include': to_list(classes),
|
'include': to_list(classes),
|
||||||
'exclude': to_list(classes),
|
'exclude': to_list(classes),
|
||||||
|
'shadow': to_list(str),
|
||||||
}}
|
}}
|
||||||
|
|
||||||
project = vs.Any(str, project_dict)
|
project = vs.Any(str, project_dict)
|
||||||
|
@ -940,6 +941,10 @@ class TenantParser(object):
|
||||||
tenant.addConfigProject(tpc)
|
tenant.addConfigProject(tpc)
|
||||||
for tpc in untrusted_tpcs:
|
for tpc in untrusted_tpcs:
|
||||||
tenant.addUntrustedProject(tpc)
|
tenant.addUntrustedProject(tpc)
|
||||||
|
|
||||||
|
for tpc in config_tpcs + untrusted_tpcs:
|
||||||
|
TenantParser._resolveShadowProjects(tenant, tpc)
|
||||||
|
|
||||||
tenant.config_projects_config, tenant.untrusted_projects_config = \
|
tenant.config_projects_config, tenant.untrusted_projects_config = \
|
||||||
TenantParser._loadTenantInRepoLayouts(merger, connections,
|
TenantParser._loadTenantInRepoLayouts(merger, connections,
|
||||||
tenant.config_projects,
|
tenant.config_projects,
|
||||||
|
@ -953,6 +958,13 @@ class TenantParser(object):
|
||||||
connections)
|
connections)
|
||||||
return tenant
|
return tenant
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _resolveShadowProjects(tenant, tpc):
|
||||||
|
shadow_projects = []
|
||||||
|
for sp in tpc.shadow_projects:
|
||||||
|
shadow_projects.append(tenant.getProject(sp)[1])
|
||||||
|
tpc.shadow_projects = frozenset(shadow_projects)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _loadProjectKeys(project_key_dir, connection_name, project):
|
def _loadProjectKeys(project_key_dir, connection_name, project):
|
||||||
project.private_key_file = (
|
project.private_key_file = (
|
||||||
|
@ -1008,9 +1020,11 @@ class TenantParser(object):
|
||||||
# Return a project object whether conf is a dict or a str
|
# Return a project object whether conf is a dict or a str
|
||||||
project = source.getProject(conf)
|
project = source.getProject(conf)
|
||||||
project_include = current_include
|
project_include = current_include
|
||||||
|
shadow_projects = []
|
||||||
else:
|
else:
|
||||||
project_name = list(conf.keys())[0]
|
project_name = list(conf.keys())[0]
|
||||||
project = source.getProject(project_name)
|
project = source.getProject(project_name)
|
||||||
|
shadow_projects = as_list(conf[project_name].get('shadow', []))
|
||||||
|
|
||||||
project_include = frozenset(
|
project_include = frozenset(
|
||||||
as_list(conf[project_name].get('include', [])))
|
as_list(conf[project_name].get('include', [])))
|
||||||
|
@ -1023,6 +1037,7 @@ class TenantParser(object):
|
||||||
|
|
||||||
tenant_project_config = model.TenantProjectConfig(project)
|
tenant_project_config = model.TenantProjectConfig(project)
|
||||||
tenant_project_config.load_classes = frozenset(project_include)
|
tenant_project_config.load_classes = frozenset(project_include)
|
||||||
|
tenant_project_config.shadow_projects = shadow_projects
|
||||||
|
|
||||||
return tenant_project_config
|
return tenant_project_config
|
||||||
|
|
||||||
|
@ -1234,7 +1249,11 @@ class TenantParser(object):
|
||||||
continue
|
continue
|
||||||
with configuration_exceptions('job', config_job):
|
with configuration_exceptions('job', config_job):
|
||||||
job = JobParser.fromYaml(tenant, layout, config_job)
|
job = JobParser.fromYaml(tenant, layout, config_job)
|
||||||
layout.addJob(job)
|
added = layout.addJob(job)
|
||||||
|
if not added:
|
||||||
|
TenantParser.log.debug(
|
||||||
|
"Skipped adding job %s which shadows an existing job" %
|
||||||
|
(job,))
|
||||||
|
|
||||||
if not skip_semaphores:
|
if not skip_semaphores:
|
||||||
for config_semaphore in data.semaphores:
|
for config_semaphore in data.semaphores:
|
||||||
|
@ -1273,13 +1292,11 @@ class TenantParser(object):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parseLayout(base, tenant, data, scheduler, connections):
|
def _parseLayout(base, tenant, data, scheduler, connections):
|
||||||
layout = model.Layout()
|
layout = model.Layout(tenant)
|
||||||
|
|
||||||
TenantParser._parseLayoutItems(layout, tenant, data,
|
TenantParser._parseLayoutItems(layout, tenant, data,
|
||||||
scheduler, connections)
|
scheduler, connections)
|
||||||
|
|
||||||
layout.tenant = tenant
|
|
||||||
|
|
||||||
for pipeline in layout.pipelines.values():
|
for pipeline in layout.pipelines.values():
|
||||||
pipeline.manager._postConfig(layout)
|
pipeline.manager._postConfig(layout)
|
||||||
|
|
||||||
|
@ -1372,7 +1389,7 @@ class ConfigLoader(object):
|
||||||
for project in tenant.untrusted_projects:
|
for project in tenant.untrusted_projects:
|
||||||
self._loadDynamicProjectData(config, project, files, False)
|
self._loadDynamicProjectData(config, project, files, False)
|
||||||
|
|
||||||
layout = model.Layout()
|
layout = model.Layout(tenant)
|
||||||
# NOTE: the actual pipeline objects (complete with queues and
|
# NOTE: the actual pipeline objects (complete with queues and
|
||||||
# enqueued items) are copied by reference here. This allows
|
# enqueued items) are copied by reference here. This allows
|
||||||
# our shadow dynamic configuration to continue to interact
|
# our shadow dynamic configuration to continue to interact
|
||||||
|
|
|
@ -2006,6 +2006,7 @@ class TenantProjectConfig(object):
|
||||||
def __init__(self, project):
|
def __init__(self, project):
|
||||||
self.project = project
|
self.project = project
|
||||||
self.load_classes = set()
|
self.load_classes = set()
|
||||||
|
self.shadow_projects = set()
|
||||||
|
|
||||||
|
|
||||||
class ProjectConfig(object):
|
class ProjectConfig(object):
|
||||||
|
@ -2129,8 +2130,8 @@ class UnparsedTenantConfig(object):
|
||||||
class Layout(object):
|
class Layout(object):
|
||||||
"""Holds all of the Pipelines."""
|
"""Holds all of the Pipelines."""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self, tenant):
|
||||||
self.tenant = None
|
self.tenant = tenant
|
||||||
self.project_configs = {}
|
self.project_configs = {}
|
||||||
self.project_templates = {}
|
self.project_templates = {}
|
||||||
self.pipelines = OrderedDict()
|
self.pipelines = OrderedDict()
|
||||||
|
@ -2159,6 +2160,18 @@ class Layout(object):
|
||||||
prior_jobs = [j for j in self.getJobs(job.name) if
|
prior_jobs = [j for j in self.getJobs(job.name) if
|
||||||
j.source_context.project !=
|
j.source_context.project !=
|
||||||
job.source_context.project]
|
job.source_context.project]
|
||||||
|
# Unless the repo is permitted to shadow another. If so, and
|
||||||
|
# the job we are adding is from a repo that is permitted to
|
||||||
|
# shadow the one with the older jobs, skip adding this job.
|
||||||
|
job_project = job.source_context.project
|
||||||
|
job_tpc = self.tenant.project_configs[job_project.canonical_name]
|
||||||
|
skip_add = False
|
||||||
|
for prior_job in prior_jobs[:]:
|
||||||
|
prior_project = prior_job.source_context.project
|
||||||
|
if prior_project in job_tpc.shadow_projects:
|
||||||
|
prior_jobs.remove(prior_job)
|
||||||
|
skip_add = True
|
||||||
|
|
||||||
if prior_jobs:
|
if prior_jobs:
|
||||||
raise Exception("Job %s in %s is not permitted to shadow "
|
raise Exception("Job %s in %s is not permitted to shadow "
|
||||||
"job %s in %s" % (
|
"job %s in %s" % (
|
||||||
|
@ -2166,11 +2179,13 @@ class Layout(object):
|
||||||
job.source_context.project,
|
job.source_context.project,
|
||||||
prior_jobs[0],
|
prior_jobs[0],
|
||||||
prior_jobs[0].source_context.project))
|
prior_jobs[0].source_context.project))
|
||||||
|
if skip_add:
|
||||||
|
return False
|
||||||
if job.name in self.jobs:
|
if job.name in self.jobs:
|
||||||
self.jobs[job.name].append(job)
|
self.jobs[job.name].append(job)
|
||||||
else:
|
else:
|
||||||
self.jobs[job.name] = [job]
|
self.jobs[job.name] = [job]
|
||||||
|
return True
|
||||||
|
|
||||||
def addNodeSet(self, nodeset):
|
def addNodeSet(self, nodeset):
|
||||||
if nodeset.name in self.nodesets:
|
if nodeset.name in self.nodesets:
|
||||||
|
|
Loading…
Reference in New Issue