Add include- and exclude-branches tenant config options

This allows operators to filter the set of branches from which
Zuul loads configuration.  They are similar to exclude-unprotected-branches
but apply to all drivers.

Change-Id: I8201b3a19efb266298decb4851430b7205e855a1

doc/source/tenants.rst

@@ -211,6 +211,27 @@ configuration. Some examples of tenant definitions are:
             exclude-unprotected-branches. This currently only affects
             GitHub and GitLab projects.
 
+         .. attr:: include-branches
+
+            A list of regexes matching branches which should be
+            processed.  If omitted, all branches are included.
+            Operates after *exclude-unprotected-branches* and so may
+            be used to further reduce the set of branches (but not
+            increase it).
+
+            It has priority over *exclude-branches*.
+
+         .. attr:: exclude-branches
+
+            A list of regexes matching branches which should be
+            processed.  If omitted, all branches are included.
+            Operates after *exclude-unprotected-branches* and so may
+            be used to further reduce the set of branches (but not
+            increase it).
+
+            It will not exclude a branch which already matched
+            *include-branches*.
+
          .. attr:: extra-config-paths
 
             Normally Zuul loads in-repo configuration from the first

releasenotes/notes/include-branches-493b6bb04414341b.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    Added new tenant project configuration options
+    attr:`tenant.untrusted-projects.<project>.include-branches` and
+    attr:`tenant.untrusted-projects.<project>.exclude-branches`.
+    Similar to *exclude-unprotected-branches*, these may be used to
+    reduce the set of branches from which Zuul will load
+    configuration.

tests/fixtures/config/tenant-parser/exclude-branches.yaml

@@ -0,0 +1,12 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project1:
+              exclude-branches:
+                - master
+                - baz
+          - org/project2

tests/fixtures/config/tenant-parser/include-branches.yaml

@@ -0,0 +1,12 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project1:
+              include-branches:
+                - foo
+                - bar
+          - org/project2

tests/unit/test_configloader.py

@@ -534,6 +534,58 @@ class TestTenantUnprotectedBranches(TenantParserTestCase):
         self.assertIsNone(tpc[project_name].exclude_unprotected_branches)
 
 
+class TestTenantIncludeBranches(TenantParserTestCase):
+    tenant_config_file = 'config/tenant-parser/include-branches.yaml'
+
+    def test_tenant_branches(self):
+        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
+
+        self.assertEqual(['common-config'],
+                         [x.name for x in tenant.config_projects])
+        self.assertEqual(['org/project1', 'org/project2'],
+                         [x.name for x in tenant.untrusted_projects])
+
+        tpc = tenant.project_configs
+        project_name = tenant.config_projects[0].canonical_name
+        self.assertEqual(['master'], tpc[project_name].branches)
+
+        # No branches pass the filter at the start
+        project_name = tenant.untrusted_projects[0].canonical_name
+        self.assertEqual([], tpc[project_name].branches)
+
+        # Create the foo branch
+        self.create_branch('org/project1', 'foo')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project1', 'foo'))
+        self.waitUntilSettled()
+
+        # It should pass the filter
+        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
+        tpc = tenant.project_configs
+        project_name = tenant.untrusted_projects[0].canonical_name
+        self.assertEqual(['foo'], tpc[project_name].branches)
+
+        # Create the baz branch
+        self.create_branch('org/project1', 'baz')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project1', 'baz'))
+        self.waitUntilSettled()
+
+        # It should not pass the filter
+        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
+        tpc = tenant.project_configs
+        project_name = tenant.untrusted_projects[0].canonical_name
+        self.assertEqual(['foo'], tpc[project_name].branches)
+
+
+class TestTenantExcludeBranches(TestTenantIncludeBranches):
+    tenant_config_file = 'config/tenant-parser/exclude-branches.yaml'
+
+    # Same test results as include-branches
+
+
 class TestTenantExcludeAll(TenantParserTestCase):
     tenant_config_file = 'config/tenant-parser/exclude-all.yaml'
 

zuul/configloader.py

@@ -1516,6 +1516,8 @@ class TenantParser(object):
         'exclude-unprotected-branches': bool,
         'extra-config-paths': no_dup_config_paths,
         'load-branch': str,
+        'include-branches': to_list(str),
+        'exclude-branches': to_list(str),
         'allow-circular-dependencies': bool,
     }}
 
@@ -1698,6 +1700,7 @@ class TenantParser(object):
             min_ltime = -1
         branches = sorted(tpc.project.source.getProjectBranches(
             tpc.project, tenant, min_ltime))
+        branches = [b for b in branches if tpc.includesBranch(b)]
         if 'master' in branches:
             branches.remove('master')
             branches = ['master'] + branches
@@ -1725,6 +1728,8 @@ class TenantParser(object):
             project_include = current_include
             shadow_projects = []
             project_exclude_unprotected_branches = None
+            project_include_branches = None
+            project_exclude_branches = None
             project_load_branch = None
         else:
             project_name = list(conf.keys())[0]
@@ -1743,6 +1748,18 @@ class TenantParser(object):
                 project_include = frozenset(project_include - project_exclude)
             project_exclude_unprotected_branches = conf[project_name].get(
                 'exclude-unprotected-branches', None)
+            project_include_branches = conf[project_name].get(
+                'include-branches', None)
+            if project_include_branches is not None:
+                project_include_branches = [
+                    re.compile(b) for b in as_list(project_include_branches)
+                ]
+            project_exclude_branches = conf[project_name].get(
+                'exclude-branches', None)
+            if project_exclude_branches is not None:
+                project_exclude_branches = [
+                    re.compile(b) for b in as_list(project_exclude_branches)
+                ]
             if conf[project_name].get('extra-config-paths') is not None:
                 extra_config_paths = as_list(
                     conf[project_name]['extra-config-paths'])
@@ -1758,6 +1775,8 @@ class TenantParser(object):
         tenant_project_config.shadow_projects = shadow_projects
         tenant_project_config.exclude_unprotected_branches = \
             project_exclude_unprotected_branches
+        tenant_project_config.include_branches = project_include_branches
+        tenant_project_config.exclude_branches = project_exclude_branches
         tenant_project_config.extra_config_files = extra_config_files
         tenant_project_config.extra_config_dirs = extra_config_dirs
         tenant_project_config.load_branch = project_load_branch

zuul/model.py

@@ -6312,6 +6312,8 @@ class TenantProjectConfig(object):
         # The tenant's default setting of exclude_unprotected_branches will
         # be overridden by this one if not None.
         self.exclude_unprotected_branches = None
+        self.include_branches = None
+        self.exclude_branches = None
         self.parsed_branch_config = {}  # branch -> ParsedConfig
         # The list of paths to look for extra zuul config files
         self.extra_config_files = ()
@@ -6320,6 +6322,28 @@ class TenantProjectConfig(object):
         # Load config from a different branch if this is a config project
         self.load_branch = None
 
+    def includesBranch(self, branch):
+        if self.include_branches is not None:
+            included = False
+            for r in self.include_branches:
+                if r.fullmatch(branch):
+                    included = True
+                    break
+        else:
+            included = True
+        if not included:
+            return False
+
+        excluded = False
+        if self.exclude_branches is not None:
+            for r in self.exclude_branches:
+                if r.fullmatch(branch):
+                    excluded = True
+                    break
+        if excluded:
+            return False
+        return True
+
 
 class ProjectPipelineConfig(ConfigObject):
     # Represents a project cofiguration in the context of a pipeline

zuul/scheduler.py

@@ -2220,6 +2220,12 @@ class Scheduler(threading.Thread):
         if tpc and not tpc.load_classes:
             reconfigure_tenant = False
 
+        # If we are listing included branches and this branch
+        # is not included, skip reconfig.
+        if (reconfigure_tenant and
+            not tpc.includesBranch(event.branch)):
+            reconfigure_tenant = False
+
         # But if the event is that branch protection status has
         # changed, do reconfigure.
         if (event.isBranchProtectionChanged()):