Cloner: add project-specific branch overrides

Add a facility to override the indicated branch for a specific
project.  In OpenStack, this is used to specify that tempest should
always use its master branch even though it has stable branches
(we no longer use stable branches for tempest).  It may also be
useful in some kinds of compatibility cross-testing.

The test for this is taken from 'test_multi_branch_project_override'
in devstack-gate.

Change-Id: I958af233d1def3b1c7362e1b2ddc77c108766358

tests/test_cloner.py

@@ -339,3 +339,87 @@ class TestCloner(ZuulTestCase):
         self.worker.hold_jobs_in_build = False
         self.worker.release()
         self.waitUntilSettled()
+
+    def test_project_override(self):
+        self.worker.hold_jobs_in_build = True
+        projects = ['org/project1', 'org/project2', 'org/project3',
+                    'org/project4', 'org/project5', 'org/project6']
+
+        self.create_branch('org/project3', 'stable/havana')
+        self.create_branch('org/project4', 'stable/havana')
+        self.create_branch('org/project6', 'stable/havana')
+        A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+        B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+        C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
+        D = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana', 'D')
+        A.addApproval('CRVW', 2)
+        B.addApproval('CRVW', 2)
+        C.addApproval('CRVW', 2)
+        D.addApproval('CRVW', 2)
+        self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
+        self.fake_gerrit.addEvent(D.addApproval('APRV', 1))
+
+        self.waitUntilSettled()
+
+        self.assertEquals(4, len(self.builds), "Four builds are running")
+
+        upstream = self.getUpstreamRepos(projects)
+        states = [
+            {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project2': str(upstream['org/project2'].commit('master')),
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[2].parameters['ZUUL_COMMIT'],
+             'org/project3': str(upstream['org/project3'].commit('master')),
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('master')),
+             },
+            {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
+             'org/project2': self.builds[2].parameters['ZUUL_COMMIT'],
+             'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
+             'org/project4': str(upstream['org/project4'].commit('master')),
+             'org/project5': str(upstream['org/project5'].commit('master')),
+             'org/project6': str(upstream['org/project6'].commit('stable/havana')),
+             },
+            ]
+
+        for number, build in enumerate(self.builds):
+            self.log.debug("Build parameters: %s", build.parameters)
+            change_number = int(build.parameters['ZUUL_CHANGE'])
+            cloner = zuul.lib.cloner.Cloner(
+                git_base_url=self.upstream_root,
+                projects=projects,
+                workspace=self.workspace_root,
+                zuul_branch=build.parameters['ZUUL_BRANCH'],
+                zuul_ref=build.parameters['ZUUL_REF'],
+                zuul_url=self.git_root,
+                project_branches={'org/project4': 'master'},
+                )
+            cloner.execute()
+            work = self.getWorkspaceRepos(projects)
+            state = states[number]
+
+            for project in projects:
+                self.assertEquals(state[project],
+                                  str(work[project].commit('HEAD')),
+                                  'Project %s commit for build %s should '
+                                  'be correct' % (project, number))
+            shutil.rmtree(self.workspace_root)
+
+        self.worker.hold_jobs_in_build = False
+        self.worker.release()
+        self.waitUntilSettled()

zuul/cmd/cloner.py

@@ -61,17 +61,24 @@ class Cloner(zuul.cmd.ZuulApp):
 
         project_env = parser.add_argument_group(
             'project tuning'
-        )
+            )
         project_env.add_argument(
             '--branch',
             help=('branch to checkout instead of Zuul selected branch, '
                   'for example to specify an alternate branch to test '
                   'client library compatibility.')
-        )
+            )
+        project_env.add_argument(
+            '--project-branch', nargs=1, action='append',
+            metavar='PROJECT=BRANCH',
+            help=('project-specific branch to checkout which takes precedence '
+                  'over --branch if it is provided; may be specified multiple '
+                  'times.')
+            )
 
         zuul_env = parser.add_argument_group(
             'zuul environnement',
-            'Let you override $ZUUL_* environnement variables.'
+            'Let you override $ZUUL_* environment variables.'
         )
         for zuul_suffix in ZUUL_ENV_SUFFIXES:
             env_name = 'ZUUL_%s' % zuul_suffix.upper()
@@ -120,6 +127,11 @@ class Cloner(zuul.cmd.ZuulApp):
     def main(self):
         self.parse_arguments()
         self.setup_logging(color=self.args.color, verbose=self.args.verbose)
+        project_branches = {}
+        if self.args.project_branch:
+            for x in self.args.project_branch:
+                project, branch = x[0].split('=')
+                project_branches[project] = branch
         cloner = zuul.lib.cloner.Cloner(
             git_base_url=self.args.git_base_url,
             projects=self.args.projects,
@@ -128,7 +140,8 @@ class Cloner(zuul.cmd.ZuulApp):
             zuul_ref=self.args.zuul_ref,
             zuul_url=self.args.zuul_url,
             branch=self.args.branch,
-            clone_map_file=self.args.clone_map_file
+            clone_map_file=self.args.clone_map_file,
+            project_branches=project_branches,
         )
         cloner.execute()
 

zuul/lib/cloner.py

@@ -28,7 +28,8 @@ class Cloner(object):
     log = logging.getLogger("zuul.Cloner")
 
     def __init__(self, git_base_url, projects, workspace, zuul_branch,
-                 zuul_ref, zuul_url, branch=None, clone_map_file=None):
+                 zuul_ref, zuul_url, branch=None, clone_map_file=None,
+                 project_branches=None):
 
         self.clone_map = []
         self.dests = None
@@ -40,6 +41,7 @@ class Cloner(object):
         self.zuul_branch = zuul_branch
         self.zuul_ref = zuul_ref
         self.zuul_url = zuul_url
+        self.project_branches = project_branches or {}
 
         if clone_map_file:
             self.readCloneMap(clone_map_file)
@@ -98,6 +100,12 @@ class Cloner(object):
          2) Zuul reference for the master branch
          3) The tip of the indicated branch
          4) The tip of the master branch
+
+        The "indicated branch" is one of the following:
+
+         A) The project-specific override branch (from project_branches arg)
+         B) The user specified branch (from the branch arg)
+         C) ZUUL_BRANCH (from the zuul_branch arg)
         """
 
         repo = self.cloneUpstream(project, dest)
@@ -107,6 +115,8 @@ class Cloner(object):
         repo.prune()
 
         indicated_branch = self.branch or self.zuul_branch
+        if project in self.project_branches:
+            indicated_branch = self.project_branches[project]
 
         override_zuul_ref = re.sub(self.zuul_branch, indicated_branch,
                                    self.zuul_ref)