Allow per-repo selection of configuration classes to load

So that multiple Zuul installations can share portions of their
configuration, allow the administrator to indicate which
configuration objects should be loaded from which repositories.

This also facilitates third-party CI, and is important for
any interaction with repos for which a given Zuul installation is
not fully responsible.

In particular, this allows an administrator to use the jobs, but
not the project-pipeline definitions from a given repo.  Or even
to use the content of a repo without reading any of the zuul
configuration therein.

Change-Id: I8a07e298c8cf4dd7cbf6f5b7fc38990f7d740af4

tests/fixtures/config/tenant-parser/git/common-config/zuul.yaml

@@ -0,0 +1,27 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+- job:
+    name: common-config-job
+
+- project:
+    name: org/project1
+    check:
+      jobs:
+        - common-config-job
+
+- project:
+    name: org/project2
+    check:
+      jobs:
+        - common-config-job

tests/fixtures/config/tenant-parser/git/org_project1/.zuul.yaml

@@ -0,0 +1,8 @@
+- job:
+    name: project1-job
+
+- project:
+    name: org/project1
+    check:
+      jobs:
+        - project1-job

tests/fixtures/config/tenant-parser/git/org_project2/.zuul.yaml

@@ -0,0 +1,8 @@
+- job:
+    name: project2-job
+
+- project:
+    name: org/project2
+    check:
+      jobs:
+        - project2-job

tests/fixtures/config/tenant-parser/groups.yaml

@@ -0,0 +1,11 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - exclude: project
+            projects:
+              - org/project1
+              - org/project2

tests/fixtures/config/tenant-parser/groups2.yaml

@@ -0,0 +1,12 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - exclude: project
+            projects:
+              - org/project1
+              - org/project2:
+                  exclude: job

tests/fixtures/config/tenant-parser/groups3.yaml

@@ -0,0 +1,14 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - include: job
+            projects:
+              - org/project1
+              - org/project2:
+                  include:
+                    - project
+                    - job

tests/fixtures/config/tenant-parser/override.yaml

@@ -0,0 +1,11 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project1:
+              exclude: project
+          - org/project2:
+              include: job

tests/fixtures/config/tenant-parser/simple.yaml

@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project1
+          - org/project2

tests/unit/test_configloader.py

@@ -0,0 +1,188 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from tests.base import ZuulTestCase
+
+
+class TenantParserTestCase(ZuulTestCase):
+    create_project_keys = True
+
+    CONFIG_SET = set(['pipeline', 'job', 'semaphore', 'project',
+                      'project-template', 'nodeset', 'secret'])
+    UNTRUSTED_SET = CONFIG_SET - set(['pipeline'])
+
+    def setupAllProjectKeys(self):
+        for project in ['common-config', 'org/project1', 'org/project2']:
+            self.setupProjectKeys('gerrit', project)
+
+
+class TestTenantSimple(TenantParserTestCase):
+    tenant_config_file = 'config/tenant-parser/simple.yaml'
+
+    def test_tenant_simple(self):
+        tenant = self.sched.abide.tenants.get('tenant-one')
+        self.assertEqual(['common-config'],
+                         [x.name for x in tenant.config_projects])
+        self.assertEqual(['org/project1', 'org/project2'],
+                         [x.name for x in tenant.untrusted_projects])
+        self.assertEqual(self.CONFIG_SET,
+                         tenant.config_projects[0].load_classes)
+        self.assertEqual(self.UNTRUSTED_SET,
+                         tenant.untrusted_projects[0].load_classes)
+        self.assertEqual(self.UNTRUSTED_SET,
+                         tenant.untrusted_projects[1].load_classes)
+        self.assertTrue('common-config-job' in tenant.layout.jobs)
+        self.assertTrue('project1-job' in tenant.layout.jobs)
+        self.assertTrue('project2-job' in tenant.layout.jobs)
+        project1_config = tenant.layout.project_configs.get(
+            'review.example.com/org/project1')
+        self.assertTrue('common-config-job' in
+                        project1_config.pipelines['check'].job_list.jobs)
+        self.assertTrue('project1-job' in
+                        project1_config.pipelines['check'].job_list.jobs)
+        project2_config = tenant.layout.project_configs.get(
+            'review.example.com/org/project2')
+        self.assertTrue('common-config-job' in
+                        project2_config.pipelines['check'].job_list.jobs)
+        self.assertTrue('project2-job' in
+                        project2_config.pipelines['check'].job_list.jobs)
+
+
+class TestTenantOverride(TenantParserTestCase):
+    tenant_config_file = 'config/tenant-parser/override.yaml'
+
+    def test_tenant_override(self):
+        tenant = self.sched.abide.tenants.get('tenant-one')
+        self.assertEqual(['common-config'],
+                         [x.name for x in tenant.config_projects])
+        self.assertEqual(['org/project1', 'org/project2'],
+                         [x.name for x in tenant.untrusted_projects])
+        self.assertEqual(self.CONFIG_SET,
+                         tenant.config_projects[0].load_classes)
+        self.assertEqual(self.UNTRUSTED_SET - set(['project']),
+                         tenant.untrusted_projects[0].load_classes)
+        self.assertEqual(set(['job']),
+                         tenant.untrusted_projects[1].load_classes)
+        self.assertTrue('common-config-job' in tenant.layout.jobs)
+        self.assertTrue('project1-job' in tenant.layout.jobs)
+        self.assertTrue('project2-job' in tenant.layout.jobs)
+        project1_config = tenant.layout.project_configs.get(
+            'review.example.com/org/project1')
+        self.assertTrue('common-config-job' in
+                        project1_config.pipelines['check'].job_list.jobs)
+        self.assertFalse('project1-job' in
+                         project1_config.pipelines['check'].job_list.jobs)
+        project2_config = tenant.layout.project_configs.get(
+            'review.example.com/org/project2')
+        self.assertTrue('common-config-job' in
+                        project2_config.pipelines['check'].job_list.jobs)
+        self.assertFalse('project2-job' in
+                         project2_config.pipelines['check'].job_list.jobs)
+
+
+class TestTenantGroups(TenantParserTestCase):
+    tenant_config_file = 'config/tenant-parser/groups.yaml'
+
+    def test_tenant_groups(self):
+        tenant = self.sched.abide.tenants.get('tenant-one')
+        self.assertEqual(['common-config'],
+                         [x.name for x in tenant.config_projects])
+        self.assertEqual(['org/project1', 'org/project2'],
+                         [x.name for x in tenant.untrusted_projects])
+        self.assertEqual(self.CONFIG_SET,
+                         tenant.config_projects[0].load_classes)
+        self.assertEqual(self.UNTRUSTED_SET - set(['project']),
+                         tenant.untrusted_projects[0].load_classes)
+        self.assertEqual(self.UNTRUSTED_SET - set(['project']),
+                         tenant.untrusted_projects[1].load_classes)
+        self.assertTrue('common-config-job' in tenant.layout.jobs)
+        self.assertTrue('project1-job' in tenant.layout.jobs)
+        self.assertTrue('project2-job' in tenant.layout.jobs)
+        project1_config = tenant.layout.project_configs.get(
+            'review.example.com/org/project1')
+        self.assertTrue('common-config-job' in
+                        project1_config.pipelines['check'].job_list.jobs)
+        self.assertFalse('project1-job' in
+                         project1_config.pipelines['check'].job_list.jobs)
+        project2_config = tenant.layout.project_configs.get(
+            'review.example.com/org/project2')
+        self.assertTrue('common-config-job' in
+                        project2_config.pipelines['check'].job_list.jobs)
+        self.assertFalse('project2-job' in
+                         project2_config.pipelines['check'].job_list.jobs)
+
+
+class TestTenantGroups2(TenantParserTestCase):
+    tenant_config_file = 'config/tenant-parser/groups2.yaml'
+
+    def test_tenant_groups2(self):
+        tenant = self.sched.abide.tenants.get('tenant-one')
+        self.assertEqual(['common-config'],
+                         [x.name for x in tenant.config_projects])
+        self.assertEqual(['org/project1', 'org/project2'],
+                         [x.name for x in tenant.untrusted_projects])
+        self.assertEqual(self.CONFIG_SET,
+                         tenant.config_projects[0].load_classes)
+        self.assertEqual(self.UNTRUSTED_SET - set(['project']),
+                         tenant.untrusted_projects[0].load_classes)
+        self.assertEqual(self.UNTRUSTED_SET - set(['project', 'job']),
+                         tenant.untrusted_projects[1].load_classes)
+        self.assertTrue('common-config-job' in tenant.layout.jobs)
+        self.assertTrue('project1-job' in tenant.layout.jobs)
+        self.assertFalse('project2-job' in tenant.layout.jobs)
+        project1_config = tenant.layout.project_configs.get(
+            'review.example.com/org/project1')
+        self.assertTrue('common-config-job' in
+                        project1_config.pipelines['check'].job_list.jobs)
+        self.assertFalse('project1-job' in
+                         project1_config.pipelines['check'].job_list.jobs)
+        project2_config = tenant.layout.project_configs.get(
+            'review.example.com/org/project2')
+        self.assertTrue('common-config-job' in
+                        project2_config.pipelines['check'].job_list.jobs)
+        self.assertFalse('project2-job' in
+                         project2_config.pipelines['check'].job_list.jobs)
+
+
+class TestTenantGroups3(TenantParserTestCase):
+    tenant_config_file = 'config/tenant-parser/groups3.yaml'
+
+    def test_tenant_groups3(self):
+        tenant = self.sched.abide.tenants.get('tenant-one')
+        self.assertEqual(['common-config'],
+                         [x.name for x in tenant.config_projects])
+        self.assertEqual(['org/project1', 'org/project2'],
+                         [x.name for x in tenant.untrusted_projects])
+        self.assertEqual(self.CONFIG_SET,
+                         tenant.config_projects[0].load_classes)
+        self.assertEqual(set(['job']),
+                         tenant.untrusted_projects[0].load_classes)
+        self.assertEqual(set(['project', 'job']),
+                         tenant.untrusted_projects[1].load_classes)
+        self.assertTrue('common-config-job' in tenant.layout.jobs)
+        self.assertTrue('project1-job' in tenant.layout.jobs)
+        self.assertTrue('project2-job' in tenant.layout.jobs)
+        project1_config = tenant.layout.project_configs.get(
+            'review.example.com/org/project1')
+        self.assertTrue('common-config-job' in
+                        project1_config.pipelines['check'].job_list.jobs)
+        self.assertFalse('project1-job' in
+                         project1_config.pipelines['check'].job_list.jobs)
+        project2_config = tenant.layout.project_configs.get(
+            'review.example.com/org/project2')
+        self.assertTrue('common-config-job' in
+                        project2_config.pipelines['check'].job_list.jobs)
+        self.assertTrue('project2-job' in
+                        project2_config.pipelines['check'].job_list.jobs)

zuul/configloader.py

@@ -859,8 +859,28 @@ class SemaphoreParser(object):
 class TenantParser(object):
     log = logging.getLogger("zuul.TenantParser")
 
-    tenant_source = vs.Schema({'config-projects': [str],
-                               'untrusted-projects': [str]})
+    classes = vs.Any('pipeline', 'job', 'semaphore', 'project',
+                     'project-template', 'nodeset', 'secret')
+
+    project_dict = {str: {
+        'include': to_list(classes),
+        'exclude': to_list(classes),
+    }}
+
+    project = vs.Any(str, project_dict)
+
+    group = {
+        'include': to_list(classes),
+        'exclude': to_list(classes),
+        vs.Required('projects'): to_list(project),
+    }
+
+    project_or_group = vs.Any(project, group)
+
+    tenant_source = vs.Schema({
+        'config-projects': to_list(project_or_group),
+        'untrusted-projects': to_list(project_or_group),
+    })
 
     @staticmethod
     def validateTenantSources(connections):
@@ -959,25 +979,85 @@ class TenantParser(object):
             (project.private_key, project.public_key) = \
                 encryption.deserialize_rsa_keypair(f.read())
 
+    @staticmethod
+    def _getProject(source, conf, current_include):
+        if isinstance(conf, six.string_types):
+            # Return a project object whether conf is a dict or a str
+            project = source.getProject(conf)
+            project_include = current_include
+        else:
+            project_name = list(conf.keys())[0]
+            project = source.getProject(project_name)
+
+            project_include = frozenset(
+                as_list(conf[project_name].get('include', [])))
+            if not project_include:
+                project_include = current_include
+            project_exclude = frozenset(
+                as_list(conf[project_name].get('exclude', [])))
+            if project_exclude:
+                project_include = frozenset(project_include - project_exclude)
+
+        project.load_classes = frozenset(project_include)
+        return project
+
+    @staticmethod
+    def _getProjects(source, conf, current_include):
+        # Return a project object whether conf is a dict or a str
+        projects = []
+        if isinstance(conf, six.string_types):
+            # A simple project name string
+            projects.append(TenantParser._getProject(
+                source, conf, current_include))
+        elif len(conf.keys()) > 1 and 'projects' in conf:
+            # This is a project group
+            if 'include' in conf:
+                current_include = set(as_list(conf['include']))
+            else:
+                current_include = current_include.copy()
+            if 'exclude' in conf:
+                exclude = set(as_list(conf['exclude']))
+                current_include = current_include - exclude
+            for project in conf['projects']:
+                sub_projects = TenantParser._getProjects(source, project,
+                                                         current_include)
+                projects.extend(sub_projects)
+        elif len(conf.keys()) == 1:
+            # A project with overrides
+            projects.append(TenantParser._getProject(
+                source, conf, current_include))
+        else:
+            raise Exception("Unable to parse project %s", conf)
+        return projects
+
     @staticmethod
     def _loadTenantProjects(project_key_dir, connections, conf_tenant):
         config_projects = []
         untrusted_projects = []
 
+        default_include = frozenset(['pipeline', 'job', 'semaphore', 'project',
+                                     'secret', 'project-template', 'nodeset'])
+
         for source_name, conf_source in conf_tenant.get('source', {}).items():
             source = connections.getSource(source_name)
 
+            current_include = default_include
             for conf_repo in conf_source.get('config-projects', []):
-                project = source.getProject(conf_repo)
-                TenantParser._loadProjectKeys(
-                    project_key_dir, source_name, project)
-                config_projects.append(project)
+                projects = TenantParser._getProjects(source, conf_repo,
+                                                     current_include)
+                for project in projects:
+                    TenantParser._loadProjectKeys(
+                        project_key_dir, source_name, project)
+                    config_projects.append(project)
 
+            current_include = frozenset(default_include - set(['pipeline']))
             for conf_repo in conf_source.get('untrusted-projects', []):
-                project = source.getProject(conf_repo)
-                TenantParser._loadProjectKeys(
-                    project_key_dir, source_name, project)
-                untrusted_projects.append(project)
+                projects = TenantParser._getProjects(source, conf_repo,
+                                                     current_include)
+                for project in projects:
+                    TenantParser._loadProjectKeys(
+                        project_key_dir, source_name, project)
+                    untrusted_projects.append(project)
 
         return config_projects, untrusted_projects
 
@@ -1090,34 +1170,78 @@ class TenantParser(object):
         return config
 
     @staticmethod
-    def _parseLayout(base, tenant, data, scheduler, connections):
-        layout = model.Layout()
-
-        for config_pipeline in data.pipelines:
-            layout.addPipeline(PipelineParser.fromYaml(layout, connections,
-                                                       scheduler,
-                                                       config_pipeline))
+    def _parseLayoutItems(layout, tenant, data, scheduler, connections,
+                          skip_pipelines=False, skip_semaphores=False):
+        if not skip_pipelines:
+            for config_pipeline in data.pipelines:
+                classes = config_pipeline['_source_context'].\
+                    project.load_classes
+                if 'pipeline' not in classes:
+                    continue
+                layout.addPipeline(PipelineParser.fromYaml(
+                    layout, connections,
+                    scheduler, config_pipeline))
 
         for config_nodeset in data.nodesets:
+            classes = config_nodeset['_source_context'].project.load_classes
+            if 'nodeset' not in classes:
+                continue
             layout.addNodeSet(NodeSetParser.fromYaml(layout, config_nodeset))
 
         for config_secret in data.secrets:
+            classes = config_secret['_source_context'].project.load_classes
+            if 'secret' not in classes:
+                continue
             layout.addSecret(SecretParser.fromYaml(layout, config_secret))
 
         for config_job in data.jobs:
+            classes = config_job['_source_context'].project.load_classes
+            if 'job' not in classes:
+                continue
             with configuration_exceptions('job', config_job):
-                layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
+                job = JobParser.fromYaml(tenant, layout, config_job)
+                layout.addJob(job)
 
-        for config_semaphore in data.semaphores:
-            layout.addSemaphore(SemaphoreParser.fromYaml(config_semaphore))
+        if not skip_semaphores:
+            for config_semaphore in data.semaphores:
+                classes = config_semaphore['_source_context'].\
+                    project.load_classes
+                if 'semaphore' not in classes:
+                    continue
+                layout.addSemaphore(SemaphoreParser.fromYaml(config_semaphore))
 
         for config_template in data.project_templates:
+            classes = config_template['_source_context'].project.load_classes
+            if 'project-template' not in classes:
+                continue
             layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
                 tenant, layout, config_template))
 
-        for config_project in data.projects.values():
+        for config_projects in data.projects.values():
+            # Unlike other config classes, we expect multiple project
+            # stanzas with the same name, so that a config repo can
+            # define a project-pipeline and the project itself can
+            # augment it.  To that end, config_project is a list of
+            # each of the project stanzas.  Each one may be (should
+            # be!) from a different repo, so filter them according to
+            # the include/exclude rules before parsing them.
+            filtered_projects = [
+                p for p in config_projects if
+                'project' in p['_source_context'].project.load_classes
+            ]
+
+            if not filtered_projects:
+                continue
+
             layout.addProjectConfig(ProjectParser.fromYaml(
-                tenant, layout, config_project))
+                tenant, layout, filtered_projects))
+
+    @staticmethod
+    def _parseLayout(base, tenant, data, scheduler, connections):
+        layout = model.Layout()
+
+        TenantParser._parseLayoutItems(layout, tenant, data,
+                                       scheduler, connections)
 
         layout.tenant = tenant
 
@@ -1228,21 +1352,8 @@ class ConfigLoader(object):
         # configuration changes.
         layout.semaphores = tenant.layout.semaphores
 
-        for config_nodeset in config.nodesets:
-            layout.addNodeSet(NodeSetParser.fromYaml(layout, config_nodeset))
+        TenantParser._parseLayoutItems(layout, tenant, config, None, None,
+                                       skip_pipelines=True,
+                                       skip_semaphores=True)
 
-        for config_secret in config.secrets:
-            layout.addSecret(SecretParser.fromYaml(layout, config_secret))
-
-        for config_job in config.jobs:
-            with configuration_exceptions('job', config_job):
-                layout.addJob(JobParser.fromYaml(tenant, layout, config_job))
-
-        for config_template in config.project_templates:
-            layout.addProjectTemplate(ProjectTemplateParser.fromYaml(
-                tenant, layout, config_template))
-
-        for config_project in config.projects.values():
-            layout.addProjectConfig(ProjectParser.fromYaml(
-                tenant, layout, config_project))
         return layout

zuul/model.py

@@ -336,6 +336,9 @@ class Project(object):
         self.foreign = foreign
         self.unparsed_config = None
         self.unparsed_branch_config = {}  # branch -> UnparsedTenantConfig
+        # Configuration object classes to include or exclude when
+        # loading zuul config files.
+        self.load_classes = frozenset()
 
     def __str__(self):
         return self.name