Add regex support to project stanzas

This adds support for regex matches of project stanzas such we can add
the same jobs to a large number of projects at once without having to
add each of them like in the following project stanza.

- job:
    name: project-common-test

- job:
    name: project-test1

- job:
    name: project-test2

- project:
    name: ^org/project.+
    check:
      jobs:
        - project-common-test

- project:
    name: org/project1
    check:
      jobs:
        - project-test1

- project:
    name: org/project2
    check:
      jobs:
        - project-test2

Change-Id: I3d9a1f22a4659c9b0c63a320798a9f19c95cc79a

doc/source/user/config.rst

@@ -1138,7 +1138,9 @@ pipeline.
       The name of the project.  If Zuul is configured with two or more
       unique projects with the same name, the canonical hostname for
       the project should be included (e.g., `git.example.com/foo`).
-      If not given it is implicitly derived from the project where this
+      This can also be a regex. In this case the regex must start with ``^``
+      and match the full project name following the same rule as name without
+      regex. If not given it is implicitly derived from the project where this
       is defined.
 
    .. attr:: templates

releasenotes/notes/project-regex-cb782f699eb10865.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Project stanzas now support regex matching of :attr:`project.name`.
+    This can be used to apply project pipelines to many projects at once.

tests/fixtures/config/regex-project/git/common-config/playbooks/project-common-test.yaml

@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []

tests/fixtures/config/regex-project/git/common-config/zuul.yaml

@@ -0,0 +1,45 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- job:
+    name: base
+    parent: null
+
+- job:
+    name: project-common-test
+    run: playbooks/project-common-test.yaml
+    nodeset:
+      nodes:
+        - name: controller
+          label: label1
+
+- job:
+    name: project-common-test-canonical
+    parent: project-common-test
+
+# This shall run project-common on org/project1 and org/project2
+# but not on org/project
+- project:
+    name: ^org/project.+
+    check:
+      jobs:
+        - project-common-test
+
+
+# This shall run project-common on org/project1 and org/project2
+# but not on org/project using canonical name matching
+- project:
+    name: ^review.*.com/org/project.+
+    check:
+      jobs:
+        - project-common-test-canonical

tests/fixtures/config/regex-project/git/org_project/README

@@ -0,0 +1 @@
+test

tests/fixtures/config/regex-project/git/org_project/playbooks/project-test.yaml

@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []

tests/fixtures/config/regex-project/git/org_project/zuul.yaml

@@ -0,0 +1,13 @@
+- job:
+    name: project-test
+    run: playbooks/project-test.yaml
+    nodeset:
+      nodes:
+        - name: controller
+          label: label1
+
+- project:
+    name: org/project
+    check:
+      jobs:
+        - project-test

tests/fixtures/config/regex-project/git/org_project1/README

@@ -0,0 +1 @@
+test

tests/fixtures/config/regex-project/git/org_project1/playbooks/project-test1.yaml

@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []

tests/fixtures/config/regex-project/git/org_project1/zuul.yaml

@@ -0,0 +1,13 @@
+- job:
+    name: project-test1
+    run: playbooks/project-test1.yaml
+    nodeset:
+      nodes:
+        - name: controller
+          label: label1
+
+- project:
+    name: org/project1
+    check:
+      jobs:
+        - project-test1

tests/fixtures/config/regex-project/git/org_project2/README

@@ -0,0 +1 @@
+test

tests/fixtures/config/regex-project/git/org_project2/playbooks/project-test2.yaml

@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []

tests/fixtures/config/regex-project/git/org_project2/zuul.yaml

@@ -0,0 +1,13 @@
+- job:
+    name: project-test2
+    run: playbooks/project-test2.yaml
+    nodeset:
+      nodes:
+        - name: controller
+          label: label1
+
+- project:
+    name: org/project2
+    check:
+      jobs:
+        - project-test2

tests/fixtures/config/regex-project/main.yaml

@@ -0,0 +1,10 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+          - org/project1
+        untrusted-projects:
+          - org/project
+          - org/project2

tests/unit/test_scheduler.py

@@ -5039,6 +5039,44 @@ class TestDuplicatePipeline(ZuulTestCase):
             self.assertIn('project-test1', A.messages[0])
 
 
+class TestSchedulerRegexProject(ZuulTestCase):
+    tenant_config_file = 'config/regex-project/main.yaml'
+
+    def test_regex_project(self):
+        "Test that changes are tested in parallel and merged in series"
+
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+        C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+        self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
+
+        self.waitUntilSettled()
+
+        # We expect the following builds:
+        #  - 1 for org/project
+        #  - 3 for org/project1
+        #  - 3 for org/project2
+        self.assertEqual(len(self.history), 7)
+        self.assertEqual(A.reported, 1)
+        self.assertEqual(B.reported, 1)
+        self.assertEqual(C.reported, 1)
+
+        self.assertHistory([
+            dict(name='project-test', result='SUCCESS', changes='1,1'),
+            dict(name='project-test1', result='SUCCESS', changes='2,1'),
+            dict(name='project-common-test', result='SUCCESS', changes='2,1'),
+            dict(name='project-common-test-canonical', result='SUCCESS',
+                 changes='2,1'),
+            dict(name='project-test2', result='SUCCESS', changes='3,1'),
+            dict(name='project-common-test', result='SUCCESS', changes='3,1'),
+            dict(name='project-common-test-canonical', result='SUCCESS',
+                 changes='3,1'),
+        ], ordered=False)
+
+
 class TestSchedulerTemplatedProject(ZuulTestCase):
     tenant_config_file = 'config/templated-project/main.yaml'
 

zuul/configloader.py

@@ -929,24 +929,43 @@ class ProjectParser(object):
             # There is no name defined so implicitly add the name
             # of the project where it is defined.
             project_name = (conf['_source_context'].project.canonical_name)
-        (trusted, project) = self.pcontext.tenant.getProject(project_name)
-        if project is None:
-            raise ProjectNotFoundError(project_name)
 
-        # Parse the project as a template since they're mostly the
-        # same.
-        project_config = self.pcontext.project_template_parser.\
-            fromYaml(conf, validate=False, freeze=False)
-        project_config.name = project.canonical_name
-
-        if not conf['_source_context'].trusted:
-            if project != conf['_source_context'].project:
+        # regex matched projects need to be validatd later
+        regex = False
+        if project_name.startswith('^'):
+            # regex matching is designed to match other projects so disallow
+            # in untrusted contexts
+            if not conf['_source_context'].trusted:
                 raise ProjectNotPermittedError()
 
-            # If this project definition is in a place where it
-            # should get implied branch matchers, set it.
-            project_config.addImpliedBranchMatcher(
-                conf['_source_context'].branch)
+            # Parse the project as a template since they're mostly the
+            # same.
+            regex = True
+            project_config = self.pcontext.project_template_parser. \
+                fromYaml(conf, validate=False, freeze=False)
+
+            project_config.name = project_name
+        else:
+            (trusted, project) = self.pcontext.tenant.getProject(project_name)
+            if project is None:
+                raise ProjectNotFoundError(project_name)
+
+            if not conf['_source_context'].trusted:
+                if project != conf['_source_context'].project:
+                    raise ProjectNotPermittedError()
+
+            # Parse the project as a template since they're mostly the
+            # same.
+            project_config = self.pcontext.project_template_parser.\
+                fromYaml(conf, validate=False, freeze=False)
+
+            project_config.name = project.canonical_name
+
+            if not conf['_source_context'].trusted:
+                # If this project definition is in a place where it
+                # should get implied branch matchers, set it.
+                project_config.addImpliedBranchMatcher(
+                    conf['_source_context'].branch)
 
         # Add templates
         for name in conf.get('templates', []):
@@ -959,7 +978,9 @@ class ProjectParser(object):
         default_branch = conf.get('default-branch', 'master')
         project_config.default_branch = default_branch
 
-        project_config.freeze()
+        # we need to freeze regex projects later
+        if not regex:
+            project_config.freeze()
         return project_config
 
 
@@ -1592,8 +1613,16 @@ class TenantParser(object):
 
         for config_project in unparsed_config.projects:
             with configuration_exceptions('project', config_project):
-                parsed_config.projects.append(
-                    pcontext.project_parser.fromYaml(config_project))
+                # we need to separate the regex projects as they are processed
+                # differently later
+                name = config_project.get('name')
+                parsed_project = pcontext.project_parser.fromYaml(
+                    config_project)
+                if name and name.startswith('^'):
+                    parsed_config.projects_by_regex.setdefault(
+                        name, []).append(parsed_project)
+                else:
+                    parsed_config.projects.append(parsed_project)
 
         return parsed_config
 
@@ -1700,6 +1729,22 @@ class TenantParser(object):
             with reference_exceptions('project-template', template):
                 layout.addProjectTemplate(template)
 
+        # The project stanzas containing a regex are separated from the normal
+        # project stanzas and organized by regex. We need to loop over each
+        # regex and copy each stanza below the regex for each matching project.
+        for regex, config_projects in parsed_config.projects_by_regex.items():
+            projects_matching_regex = tenant.getProjectsByRegex(regex)
+
+            for trusted, project in projects_matching_regex:
+                for config_project in config_projects:
+                    # we just override the project name here so a simple copy
+                    # should be enough
+                    conf = copy.copy(config_project)
+                    name = project.canonical_name
+                    conf.name = name
+                    conf.freeze()
+                    parsed_config.projects.append(conf)
+
         for project in parsed_config.projects:
             classes = self._getLoadClasses(tenant, project)
             if 'project' not in classes:

zuul/model.py

@@ -17,6 +17,7 @@ from collections import OrderedDict
 import copy
 import logging
 import os
+import re2
 import struct
 import time
 from uuid import uuid4
@@ -2733,6 +2734,7 @@ class ParsedConfig(object):
         self.jobs = []
         self.project_templates = []
         self.projects = []
+        self.projects_by_regex = {}
         self.nodesets = []
         self.secrets = []
         self.semaphores = []
@@ -2744,6 +2746,7 @@ class ParsedConfig(object):
         r.jobs = self.jobs[:]
         r.project_templates = self.project_templates[:]
         r.projects = self.projects[:]
+        r.projects_by_regex = copy.copy(self.projects_by_regex)
         r.nodesets = self.nodesets[:]
         r.secrets = self.secrets[:]
         r.semaphores = self.semaphores[:]
@@ -2759,6 +2762,8 @@ class ParsedConfig(object):
             self.nodesets.extend(conf.nodesets)
             self.secrets.extend(conf.secrets)
             self.semaphores.extend(conf.semaphores)
+            for regex, projects in conf.projects_by_regex.items():
+                self.projects_by_regex.setdefault(regex, []).extend(projects)
             return
         else:
             raise ConfigItemUnknownError()
@@ -3342,6 +3347,47 @@ class Tenant(object):
         raise Exception("Project %s is neither trusted nor untrusted" %
                         (project,))
 
+    def getProjectsByRegex(self, regex):
+        """Return all projects with a full match to either project name or
+        canonical project name.
+
+        :arg str regex: The regex to match
+        :returns: A list of tuples (trusted, project) describing the found
+            projects. Raises an exception if the same project name is found
+            several times across multiple hostnames.
+        """
+
+        matcher = re2.compile(regex)
+        projects = []
+        result = []
+
+        for name, hostname_dict in self.projects.items():
+
+            if matcher.fullmatch(name):
+                # validate that this match is unambiguous
+                values = list(hostname_dict.values())
+                if len(values) > 1:
+                    raise Exception("Project name '%s' is ambiguous, "
+                                    "please fully qualify the project "
+                                    "with a hostname. Valid hostnames "
+                                    "are %s." % (name, hostname_dict.keys()))
+                projects.append(values[0])
+            else:
+                # try to match canonical project names
+                for project in hostname_dict.values():
+                    if matcher.fullmatch(project.canonical_name):
+                        projects.append(project)
+
+        for project in projects:
+            if project in self.config_projects:
+                result.append((True, project))
+            elif project in self.untrusted_projects:
+                result.append((False, project))
+            else:
+                raise Exception("Project %s is neither trusted nor untrusted" %
+                                (project,))
+        return result
+
     def getProjectBranches(self, project):
         """Return a project's branches (filtered by this tenant config)