Allow using remote refs to find commits for change

* Before merging an item we set the remote ref to point to the current
  branch HEAD.

* Instead of deleting the 'origin' remote in the executor we just set
  it to a bogus value.

This feature enables jobs to perform quality checks on a per
commit basis (e.g. commit message checks).

For pull requests it was not possible before to determine which
commits are part of the current change.

Change-Id: I5de51917fbdfaa6ffee21dff7b63c520706c59eb

doc/source/user/jobs.rst

@@ -45,8 +45,13 @@ fallback or default branch will be used).  If your job needs to
 operate on multiple branches, simply checkout the appropriate branches
 of these git repos to ensure that the job results reflect the proposed
 future state that Zuul is testing, and all dependencies are present.
-Do not use any git remotes; the local repositories are guaranteed to
+
-be up to date.
+The git repositories will have a remote ``origin`` with refs pointing
+to the previous change in the speculative state. This means that e.g.
+a ``git diff origin/<branch>..<branch>`` will show the changes being
+tested. Note that the ``origin`` URL is set to a bogus value
+(``file:///dev/null``) and can not be used for updating the repository
+state; the local repositories are guaranteed to be up to date.
 
 The repositories will be placed on the filesystem in directories
 corresponding with the canonical hostname of their source connection.

releasenotes/notes/git-remote-refs-71bd2fc2bb05155d.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    Git repositories will have a ``origin`` remote with refs pointing to the
+    previous change in the speculative state.
+
+    This allows jobs to determine the commits that are part of a change, which
+    was not possible before. The remote URL is set to a bogus value which
+    won't work with git commands that need to talk to the remote repository.

tests/unit/test_executor.py

@@ -41,6 +41,9 @@ class TestExecutorRepos(ZuulTestCase):
                              'Project %s commit for build %s #%s should '
                              'be on the correct branch' % (
                                  project, build, number))
+            # Remote 'origin' needs to be kept intact with a bogus URL
+            self.assertEqual(repo.remotes.origin.url, 'file:///dev/null')
+            self.assertIn(state['branch'], repo.remotes.origin.refs)
         if 'commit' in state:
             self.assertEqual(state['commit'],
                              str(repo.commit('HEAD')),

tests/unit/test_merger_repo.py

@@ -76,6 +76,39 @@ class TestMergerRepo(ZuulTestCase):
             sub_repo.createRepoObject().remotes[0].url,
             message="Sub repository points to upstream project2")
 
+    def test_set_refs(self):
+        parent_path = os.path.join(self.upstream_root, 'org/project1')
+        remote_sha = self.create_commit('org/project1')
+        self.create_branch('org/project1', 'foobar')
+
+        work_repo = Repo(parent_path, self.workspace_root,
+                         'none@example.org', 'User Name', '0', '0')
+        repo = git.Repo(self.workspace_root)
+        new_sha = repo.heads.foobar.commit.hexsha
+
+        work_repo.setRefs({'refs/heads/master': new_sha}, True)
+        self.assertEqual(work_repo.getBranchHead('master').hexsha, new_sha)
+        self.assertIn('master', repo.remotes.origin.refs)
+
+        work_repo.setRefs({'refs/heads/master': remote_sha})
+        self.assertEqual(work_repo.getBranchHead('master').hexsha, remote_sha)
+        self.assertNotIn('master', repo.remotes.origin.refs)
+
+    def test_set_remote_ref(self):
+        parent_path = os.path.join(self.upstream_root, 'org/project1')
+        commit_sha = self.create_commit('org/project1')
+        self.create_commit('org/project1')
+
+        work_repo = Repo(parent_path, self.workspace_root,
+                         'none@example.org', 'User Name', '0', '0')
+        work_repo.setRemoteRef('master', commit_sha)
+        work_repo.setRemoteRef('invalid', commit_sha)
+
+        repo = git.Repo(self.workspace_root)
+        self.assertEqual(repo.remotes.origin.refs.master.commit.hexsha,
+                         commit_sha)
+        self.assertNotIn('invalid', repo.remotes.origin.refs)
+
     def test_clone_timeout(self):
         parent_path = os.path.join(self.upstream_root, 'org/project1')
         self.patch(git.Git, 'GIT_PYTHON_GIT_EXECUTABLE',

zuul/executor/server.py

@@ -764,10 +764,12 @@ class AnsibleJob(object):
             # checked out
             p = args['zuul']['projects'][project['canonical_name']]
             p['checkout'] = selected_ref
-        # Delete the origin remote from each repo we set up since
+
-        # it will not be valid within the jobs.
+        # Set the URL of the origin remote for each repo to a bogus
+        # value. Keeping the remote allows tools to use it to determine
+        # which commits are part of the current change.
         for repo in repos.values():
-            repo.deleteRemote('origin')
+            repo.setRemoteUrl('file:///dev/null')
 
         # This prepares each playbook and the roles needed for each.
         self.preparePlaybooks(args)
@@ -1864,7 +1866,8 @@ class ExecutorServer(object):
     def _getMerger(self, root, cache_root, logger=None):
         return zuul.merger.merger.Merger(
             root, self.connections, self.merge_email, self.merge_name,
-            self.merge_speed_limit, self.merge_speed_time, cache_root, logger)
+            self.merge_speed_limit, self.merge_speed_time, cache_root, logger,
+            execution_context=True)
 
     def start(self):
         self._running = True

zuul/merger/merger.py

@@ -239,7 +239,7 @@ class Repo(object):
         obj = git.objects.Object.new_from_sha(repo, binsha)
         git.refs.Reference.create(repo, path, obj, force=True)
 
-    def setRefs(self, refs):
+    def setRefs(self, refs, keep_remotes=False):
         repo = self.createRepoObject()
         current_refs = {}
         for ref in repo.refs:
@@ -248,9 +248,22 @@ class Repo(object):
         for path, hexsha in refs.items():
             self.setRef(path, hexsha, repo)
             unseen.discard(path)
+            ref = current_refs.get(path)
+            if keep_remotes and ref:
+                unseen.discard('refs/remotes/origin/{}'.format(ref.name))
         for path in unseen:
             self.deleteRef(path, repo)
 
+    def setRemoteRef(self, branch, rev):
+        repo = self.createRepoObject()
+        try:
+            origin_ref = repo.remotes.origin.refs[branch]
+        except IndexError:
+            self.log.warning("No remote ref found for branch %s", branch)
+            return
+        self.log.debug("Updating remote reference %s to %s", origin_ref, rev)
+        origin_ref.commit = rev
+
     def deleteRef(self, path, repo=None):
         if repo is None:
             repo = self.createRepoObject()
@@ -365,7 +378,8 @@ class Repo(object):
 
 class Merger(object):
     def __init__(self, working_root, connections, email, username,
-                 speed_limit, speed_time, cache_root=None, logger=None):
+                 speed_limit, speed_time, cache_root=None, logger=None,
+                 execution_context=False):
         self.logger = logger
         if logger is None:
             self.log = logging.getLogger("zuul.Merger")
@@ -381,6 +395,10 @@ class Merger(object):
         self.speed_limit = speed_limit
         self.speed_time = speed_time
         self.cache_root = cache_root
+        # Flag to determine if the merger is used for preparing repositories
+        # for job execution. This flag can be used to enable executor specific
+        # behavior e.g. to keep the 'origin' remote intact.
+        self.execution_context = execution_context
 
     def _addProject(self, hostname, project_name, url, sshkey):
         repo = None
@@ -471,7 +489,7 @@ class Merger(object):
             return
         self.log.debug("Restore repo state for project %s/%s",
                        connection_name, project_name)
-        repo.setRefs(project)
+        repo.setRefs(project, keep_remotes=self.execution_context)
 
     def _mergeChange(self, item, ref):
         repo = self.getRepo(item['connection'], item['project'])
@@ -532,6 +550,13 @@ class Merger(object):
                                 repo_state, recent)
         else:
             self.log.debug("Found base commit %s for %s" % (base, key,))
+
+        if self.execution_context:
+            # Set origin branch to the rev of the current (speculative) base.
+            # This allows tools to determine the commits that are part of a
+            # change by looking at origin/master..master.
+            repo.setRemoteRef(item['branch'], base)
+
         # Merge the change
         commit = self._mergeChange(item, base)
         if not commit: