Permit config shadowing

To support the idea that diverse zuul installations can share common
job definitions in a 'standard library' project, but still be able to
override some job definitions with their own local versions if needed,
add a tenant config option to permit a repo to shadow another one.

Place the local project first in configuration, then on the remote project,
indicate that it shadows the local one.  Then, any definitions in the
remote repository which conflict with the local will be ignored.

Change-Id: Ia715c5fa45141eacbb11449404ee3a3ec948d27f
changes/84/479084/7
James E. Blair 5 years ago
parent 08d9b7801d
commit 6459db123a
  1. 2
      tests/base.py
  2. 2
      tests/fixtures/config/shadow/git/local-config/playbooks/base.yaml
  3. 2
      tests/fixtures/config/shadow/git/local-config/playbooks/test2.yaml
  4. 25
      tests/fixtures/config/shadow/git/local-config/zuul.yaml
  5. 1
      tests/fixtures/config/shadow/git/org_project/README
  6. 10
      tests/fixtures/config/shadow/git/stdlib/.zuul.yaml
  7. 1
      tests/fixtures/config/shadow/git/stdlib/README
  8. 2
      tests/fixtures/config/shadow/git/stdlib/playbooks/base.yaml
  9. 2
      tests/fixtures/config/shadow/git/stdlib/playbooks/test1.yaml
  10. 2
      tests/fixtures/config/shadow/git/stdlib/playbooks/test2.yaml
  11. 10
      tests/fixtures/config/shadow/main.yaml
  12. 24
      tests/unit/test_model.py
  13. 14
      tests/unit/test_v3.py
  14. 27
      zuul/configloader.py
  15. 21
      zuul/model.py

@ -2139,6 +2139,8 @@ class ZuulTestCase(BaseTestCase):
# Make sure we set up an RSA key for the project so that we
# don't spend time generating one:
if isinstance(project, dict):
project = list(project.keys())[0]
key_root = os.path.join(self.state_root, 'keys')
if not os.path.isdir(key_root):
os.mkdir(key_root, 0o700)

@ -0,0 +1,25 @@
- pipeline:
name: check
manager: independent
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
verified: 1
failure:
gerrit:
verified: -1
- job:
name: base
- job:
name: test2
- project:
name: org/project
check:
jobs:
- test1
- test2

@ -0,0 +1,10 @@
- job:
name: base
- job:
name: test1
parent: base
- job:
name: test2
parent: base

@ -0,0 +1,10 @@
- tenant:
name: tenant-one
source:
gerrit:
config-projects:
- local-config
untrusted-projects:
- stdlib:
shadow: local-config
- org/project

@ -40,7 +40,7 @@ class TestJob(BaseTestCase):
self.source = Dummy(canonical_hostname='git.example.com',
connection=self.connection)
self.tenant = model.Tenant('tenant')
self.layout = model.Layout()
self.layout = model.Layout(self.tenant)
self.project = model.Project('project', self.source)
self.tpc = model.TenantProjectConfig(self.project)
self.tenant.addUntrustedProject(self.tpc)
@ -59,7 +59,7 @@ class TestJob(BaseTestCase):
@property
def job(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
layout = model.Layout(tenant)
job = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': self.context,
'_start_mark': self.start_mark,
@ -170,7 +170,7 @@ class TestJob(BaseTestCase):
def test_job_inheritance_configloader(self):
# TODO(jeblair): move this to a configloader test
tenant = model.Tenant('tenant')
layout = model.Layout()
layout = model.Layout(tenant)
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
@ -333,8 +333,8 @@ class TestJob(BaseTestCase):
'playbooks/base'])
def test_job_auth_inheritance(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
tenant = self.tenant
layout = self.layout
conf = yaml.safe_load('''
- secret:
@ -359,7 +359,7 @@ class TestJob(BaseTestCase):
secret = configloader.SecretParser.fromYaml(layout, conf)
layout.addSecret(secret)
base = configloader.JobParser.fromYaml(tenant, layout, {
base = configloader.JobParser.fromYaml(self.tenant, self.layout, {
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'base',
@ -443,7 +443,7 @@ class TestJob(BaseTestCase):
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
layout = model.Layout(tenant)
tpc = model.TenantProjectConfig(self.project)
tenant.addUntrustedProject(tpc)
@ -520,7 +520,7 @@ class TestJob(BaseTestCase):
def test_inheritance_keeps_matchers(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
layout = model.Layout(tenant)
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
@ -571,11 +571,13 @@ class TestJob(BaseTestCase):
self.assertEqual([], item.getJobs())
def test_job_source_project(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
tenant = self.tenant
layout = self.layout
base_project = model.Project('base_project', self.source)
base_context = model.SourceContext(base_project, 'master',
'test', True)
tpc = model.TenantProjectConfig(base_project)
tenant.addUntrustedProject(tpc)
base = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': base_context,
@ -587,6 +589,8 @@ class TestJob(BaseTestCase):
other_project = model.Project('other_project', self.source)
other_context = model.SourceContext(other_project, 'master',
'test', True)
tpc = model.TenantProjectConfig(other_project)
tenant.addUntrustedProject(tpc)
base2 = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': other_context,
'_start_mark': self.start_mark,

@ -617,3 +617,17 @@ class TestRoles(ZuulTestCase):
self.assertHistory([
dict(name='project-test', result='SUCCESS', changes='1,1 2,1'),
])
class TestShadow(ZuulTestCase):
tenant_config_file = 'config/shadow/main.yaml'
def test_shadow(self):
# Test that a repo is allowed to shadow another's job definitions.
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertHistory([
dict(name='test1', result='SUCCESS', changes='1,1'),
dict(name='test2', result='SUCCESS', changes='1,1'),
])

@ -887,6 +887,7 @@ class TenantParser(object):
project_dict = {str: {
'include': to_list(classes),
'exclude': to_list(classes),
'shadow': to_list(str),
}}
project = vs.Any(str, project_dict)
@ -940,6 +941,10 @@ class TenantParser(object):
tenant.addConfigProject(tpc)
for tpc in untrusted_tpcs:
tenant.addUntrustedProject(tpc)
for tpc in config_tpcs + untrusted_tpcs:
TenantParser._resolveShadowProjects(tenant, tpc)
tenant.config_projects_config, tenant.untrusted_projects_config = \
TenantParser._loadTenantInRepoLayouts(merger, connections,
tenant.config_projects,
@ -953,6 +958,13 @@ class TenantParser(object):
connections)
return tenant
@staticmethod
def _resolveShadowProjects(tenant, tpc):
shadow_projects = []
for sp in tpc.shadow_projects:
shadow_projects.append(tenant.getProject(sp)[1])
tpc.shadow_projects = frozenset(shadow_projects)
@staticmethod
def _loadProjectKeys(project_key_dir, connection_name, project):
project.private_key_file = (
@ -1008,9 +1020,11 @@ class TenantParser(object):
# Return a project object whether conf is a dict or a str
project = source.getProject(conf)
project_include = current_include
shadow_projects = []
else:
project_name = list(conf.keys())[0]
project = source.getProject(project_name)
shadow_projects = as_list(conf[project_name].get('shadow', []))
project_include = frozenset(
as_list(conf[project_name].get('include', [])))
@ -1023,6 +1037,7 @@ class TenantParser(object):
tenant_project_config = model.TenantProjectConfig(project)
tenant_project_config.load_classes = frozenset(project_include)
tenant_project_config.shadow_projects = shadow_projects
return tenant_project_config
@ -1234,7 +1249,11 @@ class TenantParser(object):
continue
with configuration_exceptions('job', config_job):
job = JobParser.fromYaml(tenant, layout, config_job)
layout.addJob(job)
added = layout.addJob(job)
if not added:
TenantParser.log.debug(
"Skipped adding job %s which shadows an existing job" %
(job,))
if not skip_semaphores:
for config_semaphore in data.semaphores:
@ -1273,13 +1292,11 @@ class TenantParser(object):
@staticmethod
def _parseLayout(base, tenant, data, scheduler, connections):
layout = model.Layout()
layout = model.Layout(tenant)
TenantParser._parseLayoutItems(layout, tenant, data,
scheduler, connections)
layout.tenant = tenant
for pipeline in layout.pipelines.values():
pipeline.manager._postConfig(layout)
@ -1372,7 +1389,7 @@ class ConfigLoader(object):
for project in tenant.untrusted_projects:
self._loadDynamicProjectData(config, project, files, False)
layout = model.Layout()
layout = model.Layout(tenant)
# NOTE: the actual pipeline objects (complete with queues and
# enqueued items) are copied by reference here. This allows
# our shadow dynamic configuration to continue to interact

@ -2006,6 +2006,7 @@ class TenantProjectConfig(object):
def __init__(self, project):
self.project = project
self.load_classes = set()
self.shadow_projects = set()
class ProjectConfig(object):
@ -2129,8 +2130,8 @@ class UnparsedTenantConfig(object):
class Layout(object):
"""Holds all of the Pipelines."""
def __init__(self):
self.tenant = None
def __init__(self, tenant):
self.tenant = tenant
self.project_configs = {}
self.project_templates = {}
self.pipelines = OrderedDict()
@ -2159,6 +2160,18 @@ class Layout(object):
prior_jobs = [j for j in self.getJobs(job.name) if
j.source_context.project !=
job.source_context.project]
# Unless the repo is permitted to shadow another. If so, and
# the job we are adding is from a repo that is permitted to
# shadow the one with the older jobs, skip adding this job.
job_project = job.source_context.project
job_tpc = self.tenant.project_configs[job_project.canonical_name]
skip_add = False
for prior_job in prior_jobs[:]:
prior_project = prior_job.source_context.project
if prior_project in job_tpc.shadow_projects:
prior_jobs.remove(prior_job)
skip_add = True
if prior_jobs:
raise Exception("Job %s in %s is not permitted to shadow "
"job %s in %s" % (
@ -2166,11 +2179,13 @@ class Layout(object):
job.source_context.project,
prior_jobs[0],
prior_jobs[0].source_context.project))
if skip_add:
return False
if job.name in self.jobs:
self.jobs[job.name].append(job)
else:
self.jobs[job.name] = [job]
return True
def addNodeSet(self, nodeset):
if nodeset.name in self.nodesets:

Loading…
Cancel
Save