diff --git a/tests/base.py b/tests/base.py index e605c87baa..696156f844 100755 --- a/tests/base.py +++ b/tests/base.py @@ -2139,6 +2139,8 @@ class ZuulTestCase(BaseTestCase): # Make sure we set up an RSA key for the project so that we # don't spend time generating one: + if isinstance(project, dict): + project = list(project.keys())[0] key_root = os.path.join(self.state_root, 'keys') if not os.path.isdir(key_root): os.mkdir(key_root, 0o700) diff --git a/tests/fixtures/config/shadow/git/local-config/playbooks/base.yaml b/tests/fixtures/config/shadow/git/local-config/playbooks/base.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/shadow/git/local-config/playbooks/base.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/shadow/git/local-config/playbooks/test2.yaml b/tests/fixtures/config/shadow/git/local-config/playbooks/test2.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/shadow/git/local-config/playbooks/test2.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/shadow/git/local-config/zuul.yaml b/tests/fixtures/config/shadow/git/local-config/zuul.yaml new file mode 100644 index 0000000000..756e8438f9 --- /dev/null +++ b/tests/fixtures/config/shadow/git/local-config/zuul.yaml @@ -0,0 +1,25 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + verified: 1 + failure: + gerrit: + verified: -1 + +- job: + name: base + +- job: + name: test2 + +- project: + name: org/project + check: + jobs: + - test1 + - test2 diff --git a/tests/fixtures/config/shadow/git/org_project/README b/tests/fixtures/config/shadow/git/org_project/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/shadow/git/org_project/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml b/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml new file mode 100644 index 0000000000..6a6f9c9b5a --- /dev/null +++ b/tests/fixtures/config/shadow/git/stdlib/.zuul.yaml @@ -0,0 +1,10 @@ +- job: + name: base + +- job: + name: test1 + parent: base + +- job: + name: test2 + parent: base diff --git a/tests/fixtures/config/shadow/git/stdlib/README b/tests/fixtures/config/shadow/git/stdlib/README new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/fixtures/config/shadow/git/stdlib/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/shadow/git/stdlib/playbooks/base.yaml b/tests/fixtures/config/shadow/git/stdlib/playbooks/base.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/shadow/git/stdlib/playbooks/base.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/shadow/git/stdlib/playbooks/test1.yaml b/tests/fixtures/config/shadow/git/stdlib/playbooks/test1.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/shadow/git/stdlib/playbooks/test1.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/shadow/git/stdlib/playbooks/test2.yaml b/tests/fixtures/config/shadow/git/stdlib/playbooks/test2.yaml new file mode 100644 index 0000000000..f679dceaef --- /dev/null +++ b/tests/fixtures/config/shadow/git/stdlib/playbooks/test2.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/shadow/main.yaml b/tests/fixtures/config/shadow/main.yaml new file mode 100644 index 0000000000..f148a84e90 --- /dev/null +++ b/tests/fixtures/config/shadow/main.yaml @@ -0,0 +1,10 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - local-config + untrusted-projects: + - stdlib: + shadow: local-config + - org/project diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index 3ab3305965..7fe101e903 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -40,7 +40,7 @@ class TestJob(BaseTestCase): self.source = Dummy(canonical_hostname='git.example.com', connection=self.connection) self.tenant = model.Tenant('tenant') - self.layout = model.Layout() + self.layout = model.Layout(self.tenant) self.project = model.Project('project', self.source) self.tpc = model.TenantProjectConfig(self.project) self.tenant.addUntrustedProject(self.tpc) @@ -59,7 +59,7 @@ class TestJob(BaseTestCase): @property def job(self): tenant = model.Tenant('tenant') - layout = model.Layout() + layout = model.Layout(tenant) job = configloader.JobParser.fromYaml(tenant, layout, { '_source_context': self.context, '_start_mark': self.start_mark, @@ -170,7 +170,7 @@ class TestJob(BaseTestCase): def test_job_inheritance_configloader(self): # TODO(jeblair): move this to a configloader test tenant = model.Tenant('tenant') - layout = model.Layout() + layout = model.Layout(tenant) pipeline = model.Pipeline('gate', layout) layout.addPipeline(pipeline) @@ -333,8 +333,8 @@ class TestJob(BaseTestCase): 'playbooks/base']) def test_job_auth_inheritance(self): - tenant = model.Tenant('tenant') - layout = model.Layout() + tenant = self.tenant + layout = self.layout conf = yaml.safe_load(''' - secret: @@ -359,7 +359,7 @@ class TestJob(BaseTestCase): secret = configloader.SecretParser.fromYaml(layout, conf) layout.addSecret(secret) - base = configloader.JobParser.fromYaml(tenant, layout, { + base = configloader.JobParser.fromYaml(self.tenant, self.layout, { '_source_context': self.context, '_start_mark': self.start_mark, 'name': 'base', @@ -443,7 +443,7 @@ class TestJob(BaseTestCase): def test_job_inheritance_job_tree(self): tenant = model.Tenant('tenant') - layout = model.Layout() + layout = model.Layout(tenant) tpc = model.TenantProjectConfig(self.project) tenant.addUntrustedProject(tpc) @@ -520,7 +520,7 @@ class TestJob(BaseTestCase): def test_inheritance_keeps_matchers(self): tenant = model.Tenant('tenant') - layout = model.Layout() + layout = model.Layout(tenant) pipeline = model.Pipeline('gate', layout) layout.addPipeline(pipeline) @@ -571,11 +571,13 @@ class TestJob(BaseTestCase): self.assertEqual([], item.getJobs()) def test_job_source_project(self): - tenant = model.Tenant('tenant') - layout = model.Layout() + tenant = self.tenant + layout = self.layout base_project = model.Project('base_project', self.source) base_context = model.SourceContext(base_project, 'master', 'test', True) + tpc = model.TenantProjectConfig(base_project) + tenant.addUntrustedProject(tpc) base = configloader.JobParser.fromYaml(tenant, layout, { '_source_context': base_context, @@ -587,6 +589,8 @@ class TestJob(BaseTestCase): other_project = model.Project('other_project', self.source) other_context = model.SourceContext(other_project, 'master', 'test', True) + tpc = model.TenantProjectConfig(other_project) + tenant.addUntrustedProject(tpc) base2 = configloader.JobParser.fromYaml(tenant, layout, { '_source_context': other_context, '_start_mark': self.start_mark, diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index 7c5fa7069a..f765a5339c 100644 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -617,3 +617,17 @@ class TestRoles(ZuulTestCase): self.assertHistory([ dict(name='project-test', result='SUCCESS', changes='1,1 2,1'), ]) + + +class TestShadow(ZuulTestCase): + tenant_config_file = 'config/shadow/main.yaml' + + def test_shadow(self): + # Test that a repo is allowed to shadow another's job definitions. + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + self.assertHistory([ + dict(name='test1', result='SUCCESS', changes='1,1'), + dict(name='test2', result='SUCCESS', changes='1,1'), + ]) diff --git a/zuul/configloader.py b/zuul/configloader.py index 18fca83d6e..256a859eb5 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -887,6 +887,7 @@ class TenantParser(object): project_dict = {str: { 'include': to_list(classes), 'exclude': to_list(classes), + 'shadow': to_list(str), }} project = vs.Any(str, project_dict) @@ -940,6 +941,10 @@ class TenantParser(object): tenant.addConfigProject(tpc) for tpc in untrusted_tpcs: tenant.addUntrustedProject(tpc) + + for tpc in config_tpcs + untrusted_tpcs: + TenantParser._resolveShadowProjects(tenant, tpc) + tenant.config_projects_config, tenant.untrusted_projects_config = \ TenantParser._loadTenantInRepoLayouts(merger, connections, tenant.config_projects, @@ -953,6 +958,13 @@ class TenantParser(object): connections) return tenant + @staticmethod + def _resolveShadowProjects(tenant, tpc): + shadow_projects = [] + for sp in tpc.shadow_projects: + shadow_projects.append(tenant.getProject(sp)[1]) + tpc.shadow_projects = frozenset(shadow_projects) + @staticmethod def _loadProjectKeys(project_key_dir, connection_name, project): project.private_key_file = ( @@ -1008,9 +1020,11 @@ class TenantParser(object): # Return a project object whether conf is a dict or a str project = source.getProject(conf) project_include = current_include + shadow_projects = [] else: project_name = list(conf.keys())[0] project = source.getProject(project_name) + shadow_projects = as_list(conf[project_name].get('shadow', [])) project_include = frozenset( as_list(conf[project_name].get('include', []))) @@ -1023,6 +1037,7 @@ class TenantParser(object): tenant_project_config = model.TenantProjectConfig(project) tenant_project_config.load_classes = frozenset(project_include) + tenant_project_config.shadow_projects = shadow_projects return tenant_project_config @@ -1234,7 +1249,11 @@ class TenantParser(object): continue with configuration_exceptions('job', config_job): job = JobParser.fromYaml(tenant, layout, config_job) - layout.addJob(job) + added = layout.addJob(job) + if not added: + TenantParser.log.debug( + "Skipped adding job %s which shadows an existing job" % + (job,)) if not skip_semaphores: for config_semaphore in data.semaphores: @@ -1273,13 +1292,11 @@ class TenantParser(object): @staticmethod def _parseLayout(base, tenant, data, scheduler, connections): - layout = model.Layout() + layout = model.Layout(tenant) TenantParser._parseLayoutItems(layout, tenant, data, scheduler, connections) - layout.tenant = tenant - for pipeline in layout.pipelines.values(): pipeline.manager._postConfig(layout) @@ -1372,7 +1389,7 @@ class ConfigLoader(object): for project in tenant.untrusted_projects: self._loadDynamicProjectData(config, project, files, False) - layout = model.Layout() + layout = model.Layout(tenant) # NOTE: the actual pipeline objects (complete with queues and # enqueued items) are copied by reference here. This allows # our shadow dynamic configuration to continue to interact diff --git a/zuul/model.py b/zuul/model.py index 3cf498431c..17301b70b9 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -2006,6 +2006,7 @@ class TenantProjectConfig(object): def __init__(self, project): self.project = project self.load_classes = set() + self.shadow_projects = set() class ProjectConfig(object): @@ -2129,8 +2130,8 @@ class UnparsedTenantConfig(object): class Layout(object): """Holds all of the Pipelines.""" - def __init__(self): - self.tenant = None + def __init__(self, tenant): + self.tenant = tenant self.project_configs = {} self.project_templates = {} self.pipelines = OrderedDict() @@ -2159,6 +2160,18 @@ class Layout(object): prior_jobs = [j for j in self.getJobs(job.name) if j.source_context.project != job.source_context.project] + # Unless the repo is permitted to shadow another. If so, and + # the job we are adding is from a repo that is permitted to + # shadow the one with the older jobs, skip adding this job. + job_project = job.source_context.project + job_tpc = self.tenant.project_configs[job_project.canonical_name] + skip_add = False + for prior_job in prior_jobs[:]: + prior_project = prior_job.source_context.project + if prior_project in job_tpc.shadow_projects: + prior_jobs.remove(prior_job) + skip_add = True + if prior_jobs: raise Exception("Job %s in %s is not permitted to shadow " "job %s in %s" % ( @@ -2166,11 +2179,13 @@ class Layout(object): job.source_context.project, prior_jobs[0], prior_jobs[0].source_context.project)) - + if skip_add: + return False if job.name in self.jobs: self.jobs[job.name].append(job) else: self.jobs[job.name] = [job] + return True def addNodeSet(self, nodeset): if nodeset.name in self.nodesets: