Merge "Use override-checkout to select jobs"

doc/source/user/config.rst

@@ -692,6 +692,11 @@ Here is an example of two job definitions:
       attribute to apply this behavior to a subset of a job's
       projects.
 
+      This value is also used to help select which variants of a job
+      to run.  If ``override-checkout`` is set, then Zuul will use
+      this value instead of the branch of the item being tested when
+      collecting jobs to run.
+
    .. attr:: timeout
 
       The time in seconds that the job should be allowed to run before
@@ -837,6 +842,12 @@ Here is an example of two job definitions:
          :attr:`job.override-checkout` attribute to apply the same
          behavior to all projects in a job.
 
+         This value is also used to help select which variants of a
+         job to run.  If ``override-checkout`` is set, then Zuul will
+         use this value instead of the branch of the item being tested
+         when collecting any jobs to run which are defined in this
+         project.
+
    .. attr:: vars
 
       A dictionary of variables to supply to Ansible.  When inheriting
@@ -895,6 +906,12 @@ Here is an example of two job definitions:
       branch of an item, then that job is not run for the item.
       Otherwise, all of the job variants which match that branch (and
       any other selection criteria) are used when freezing the job.
+      However, if :attr:`job.override-checkout` or
+      :attr:`job.required-projects.override-checkout` are set for a
+      project, Zuul will attempt to use the job variants which match
+      the values supplied in ``override-checkout`` for jobs defined in
+      those projects.  This can be used to run a job defined in one
+      project on another project without a matching branch.
 
       This example illustrates a job called *run-tests* which uses a
       nodeset based on the current release of an operating system to

tests/fixtures/config/branch-mismatch/git/common-config/playbooks/base.yaml

@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []

tests/fixtures/config/branch-mismatch/git/common-config/zuul.yaml

@@ -0,0 +1,22 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+
+- project:
+    name: common-config
+    check:
+      jobs: []

tests/fixtures/config/branch-mismatch/git/org_project1/README

@@ -0,0 +1 @@
+test

tests/fixtures/config/branch-mismatch/git/org_project1/zuul.yaml

@@ -0,0 +1,7 @@
+- job:
+    name: project-test1
+
+- project:
+    check:
+      jobs:
+        - project-test1

tests/fixtures/config/branch-mismatch/git/org_project2/README

@@ -0,0 +1 @@
+test

tests/fixtures/config/branch-mismatch/git/org_project2/zuul.yaml

@@ -0,0 +1,13 @@
+- job:
+    name: project-test2
+    parent: project-test1
+    override-checkout: stable
+
+- project:
+    check:
+      jobs:
+        - project-test1:
+            required-projects:
+              - name: org/project1
+                override-checkout: stable
+        - project-test2

tests/fixtures/config/branch-mismatch/main.yaml

@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project1
+          - org/project2

tests/unit/test_v3.py

@@ -497,6 +497,42 @@ class TestBranchVariants(ZuulTestCase):
         self.waitUntilSettled()
 
 
+class TestBranchMismatch(ZuulTestCase):
+    tenant_config_file = 'config/branch-mismatch/main.yaml'
+
+    def test_job_override_branch(self):
+        "Test that override-checkout overrides branch matchers as well"
+
+        # Make sure the parent job repo is branched, so it gets
+        # implied branch matchers.
+        self.create_branch('org/project1', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project1', 'stable'))
+
+        # The child job repo should have a branch which does not exist
+        # in the parent job repo.
+        self.create_branch('org/project2', 'devel')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project2', 'devel'))
+
+        # A job in a repo with a weird branch name should use the
+        # parent job from the parent job's master (default) branch.
+        A = self.fake_gerrit.addFakeChange('org/project2', 'devel', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+        # project-test2 should run because it inherits from
+        # project-test1 and we will use the fallback branch to find
+        # project-test1 variants, but project-test1 itself, even
+        # though it is in the project-pipeline config, should not run
+        # because it doesn't directly match.
+        self.assertHistory([
+            dict(name='project-test1', result='SUCCESS', changes='1,1'),
+            dict(name='project-test2', result='SUCCESS', changes='1,1'),
+        ], ordered=False)
+
+
 class TestCentralJobs(ZuulTestCase):
     tenant_config_file = 'config/central-jobs/main.yaml'
 

zuul/configloader.py

@@ -700,10 +700,10 @@ class JobParser(object):
                 (trusted, project) = tenant.getProject(project_name)
                 if project is None:
                     raise Exception("Unknown project %s" % (project_name,))
-                job_project = model.JobProject(project_name,
+                job_project = model.JobProject(project.canonical_name,
                                                project_override_branch,
                                                project_override_checkout)
-                new_projects[project_name] = job_project
+                new_projects[project.canonical_name] = job_project
             job.required_projects = new_projects
 
         tags = conf.get('tags')

zuul/model.py

@@ -1108,8 +1108,18 @@ class Job(object):
 
         self.inheritance_path = self.inheritance_path + (repr(other),)
 
-    def changeMatches(self, change):
-        if self.branch_matcher and not self.branch_matcher.matches(change):
+    def changeMatches(self, change, override_branch=None):
+        if override_branch is None:
+            branch_change = change
+        else:
+            # If an override branch is supplied, create a very basic
+            # change (a Ref) and set its branch to the override
+            # branch.
+            branch_change = Ref(change.project)
+            branch_change.ref = override_branch
+
+        if self.branch_matcher and not self.branch_matcher.matches(
+                branch_change):
             return False
 
         if self.file_matcher and not self.file_matcher.matches(change):
@@ -2071,9 +2081,6 @@ class Ref(object):
     def isUpdateOf(self, other):
         return False
 
-    def filterJobs(self, jobs):
-        return filter(lambda job: job.changeMatches(self), jobs)
-
     def getRelatedChanges(self):
         return set()
 
@@ -2668,21 +2675,33 @@ class Layout(object):
     def addProjectConfig(self, project_config):
         self.project_configs[project_config.name] = project_config
 
-    def collectJobs(self, item, jobname, change, path=None, jobs=None,
-                    stack=None):
-        if stack is None:
-            stack = []
-        if jobs is None:
-            jobs = []
-        if path is None:
-            path = []
-        path.append(jobname)
+    def _updateOverrideCheckouts(self, override_checkouts, job):
+        # Update the values in an override_checkouts dict with those
+        # in a job.  Used in collectJobVariants.
+        if job.override_checkout:
+            override_checkouts[None] = job.override_checkout
+        for req in job.required_projects.values():
+            if req.override_checkout:
+                override_checkouts[req.project_name] = req.override_checkout
+
+    def _collectJobVariants(self, item, jobname, change, path, jobs, stack,
+                            override_checkouts, indent):
         matched = False
-        indent = len(path) + 1
-        item.debug("Collecting job variants for {jobname}".format(
-            jobname=jobname), indent=indent)
+        local_override_checkouts = override_checkouts.copy()
+        override_branch = None
+        project = None
         for variant in self.getJobs(jobname):
-            if not variant.changeMatches(change):
+            if project is None and variant.source_context:
+                project = variant.source_context.project
+                if override_checkouts.get(None) is not None:
+                    override_branch = override_checkouts.get(None)
+                override_branch = override_checkouts.get(
+                    project.canonical_name, override_branch)
+                branches = self.tenant.getProjectBranches(project)
+                if override_branch not in branches:
+                    override_branch = None
+            if not variant.changeMatches(change,
+                                         override_branch=override_branch):
                 self.log.debug("Variant %s did not match %s", repr(variant),
                                change)
                 item.debug("Variant {variant} did not match".format(
@@ -2698,17 +2717,53 @@ class Layout(object):
                     parent = self.tenant.default_base_job
             else:
                 parent = None
+            self._updateOverrideCheckouts(local_override_checkouts, variant)
             if parent and parent not in path:
                 if parent in stack:
                     raise Exception("Dependency cycle in jobs: %s" % stack)
                 self.collectJobs(item, parent, change, path, jobs,
-                                 stack + [jobname])
+                                 stack + [jobname], local_override_checkouts)
             matched = True
-            jobs.append(variant)
+            if variant not in jobs:
+                jobs.append(variant)
+        return matched
+
+    def collectJobs(self, item, jobname, change, path=None, jobs=None,
+                    stack=None, override_checkouts=None):
+        # Stack is the recursion stack of job parent names.  Each time
+        # we go up a level, we add to stack, and it's popped as we
+        # descend.
+        if stack is None:
+            stack = []
+        # Jobs is the list of jobs we've accumulated.
+        if jobs is None:
+            jobs = []
+        # Path is the list of job names we've examined.  It
+        # accumulates and never reduces.  If more than one job has the
+        # same parent, this will prevent us from adding it a second
+        # time.
+        if path is None:
+            path = []
+        # Override_checkouts is a dictionary of canonical project
+        # names -> branch names.  It is not mutated, but instead new
+        # copies are made and updated as we ascend the hierarchy, so
+        # higher levels don't affect lower levels after we descend.
+        # It's used to override the branch matchers for jobs.
+        if override_checkouts is None:
+            override_checkouts = {}
+        path.append(jobname)
+        matched = False
+        indent = len(path) + 1
+        msg = "Collecting job variants for {jobname}".format(jobname=jobname)
+        self.log.debug(msg)
+        item.debug(msg, indent=indent)
+        matched = self._collectJobVariants(
+            item, jobname, change, path, jobs, stack, override_checkouts,
+            indent)
         if not matched:
             self.log.debug("No matching parents for job %s and change %s",
                            jobname, change)
-            item.debug("No matching parent for {jobname}".format(
+            item.debug("No matching parents for {jobname}".format(
                 jobname=repr(jobname)), indent=indent)
             raise NoMatchingParentError()
         return jobs
@@ -2723,8 +2778,17 @@ class Layout(object):
             self.log.debug("Collecting jobs %s for %s", jobname, change)
             item.debug("Freezing job {jobname}".format(
                 jobname=jobname), indent=1)
+            # Create the initial list of override_checkouts, which are
+            # used as we walk up the hierarchy to expand the set of
+            # jobs which match.
+            override_checkouts = {}
+            for variant in job_list.jobs[jobname]:
+                if variant.changeMatches(change):
+                    self._updateOverrideCheckouts(override_checkouts, variant)
             try:
-                variants = self.collectJobs(item, jobname, change)
+                variants = self.collectJobs(
+                    item, jobname, change,
+                    override_checkouts=override_checkouts)
             except NoMatchingParentError:
                 variants = None
             if not variants: