Browse Source

Allow using remote refs to find commits for change

* Before merging an item we set the remote ref to point to the current
  branch HEAD.

* Instead of deleting the 'origin' remote in the executor we just set
  it to a bogus value.

This feature enables jobs to perform quality checks on a per
commit basis (e.g. commit message checks).

For pull requests it was not possible before to determine which
commits are part of the current change.

Change-Id: I5de51917fbdfaa6ffee21dff7b63c520706c59eb
Simon Westphahl 3 years ago
6 changed files with 87 additions and 9 deletions
  1. +7
  2. +9
  3. +3
  4. +33
  5. +7
  6. +28

+ 7
- 2
doc/source/user/jobs.rst View File

@ -45,8 +45,13 @@ fallback or default branch will be used). If your job needs to
operate on multiple branches, simply checkout the appropriate branches
of these git repos to ensure that the job results reflect the proposed
future state that Zuul is testing, and all dependencies are present.
Do not use any git remotes; the local repositories are guaranteed to
be up to date.
The git repositories will have a remote ``origin`` with refs pointing
to the previous change in the speculative state. This means that e.g.
a ``git diff origin/<branch>..<branch>`` will show the changes being
tested. Note that the ``origin`` URL is set to a bogus value
(``file:///dev/null``) and can not be used for updating the repository
state; the local repositories are guaranteed to be up to date.
The repositories will be placed on the filesystem in directories
corresponding with the canonical hostname of their source connection.

+ 9
- 0
releasenotes/notes/git-remote-refs-71bd2fc2bb05155d.yaml View File

@ -0,0 +1,9 @@
- |
Git repositories will have a ``origin`` remote with refs pointing to the
previous change in the speculative state.
This allows jobs to determine the commits that are part of a change, which
was not possible before. The remote URL is set to a bogus value which
won't work with git commands that need to talk to the remote repository.

+ 3
- 0
tests/unit/ View File

@ -41,6 +41,9 @@ class TestExecutorRepos(ZuulTestCase):
'Project %s commit for build %s #%s should '
'be on the correct branch' % (
project, build, number))
# Remote 'origin' needs to be kept intact with a bogus URL
self.assertEqual(repo.remotes.origin.url, 'file:///dev/null')
self.assertIn(state['branch'], repo.remotes.origin.refs)
if 'commit' in state:

+ 33
- 0
tests/unit/ View File

@ -76,6 +76,39 @@ class TestMergerRepo(ZuulTestCase):
message="Sub repository points to upstream project2")
def test_set_refs(self):
parent_path = os.path.join(self.upstream_root, 'org/project1')
remote_sha = self.create_commit('org/project1')
self.create_branch('org/project1', 'foobar')
work_repo = Repo(parent_path, self.workspace_root,
'', 'User Name', '0', '0')
repo = git.Repo(self.workspace_root)
new_sha = repo.heads.foobar.commit.hexsha
work_repo.setRefs({'refs/heads/master': new_sha}, True)
self.assertEqual(work_repo.getBranchHead('master').hexsha, new_sha)
self.assertIn('master', repo.remotes.origin.refs)
work_repo.setRefs({'refs/heads/master': remote_sha})
self.assertEqual(work_repo.getBranchHead('master').hexsha, remote_sha)
self.assertNotIn('master', repo.remotes.origin.refs)
def test_set_remote_ref(self):
parent_path = os.path.join(self.upstream_root, 'org/project1')
commit_sha = self.create_commit('org/project1')
work_repo = Repo(parent_path, self.workspace_root,
'', 'User Name', '0', '0')
work_repo.setRemoteRef('master', commit_sha)
work_repo.setRemoteRef('invalid', commit_sha)
repo = git.Repo(self.workspace_root)
self.assertNotIn('invalid', repo.remotes.origin.refs)
def test_clone_timeout(self):
parent_path = os.path.join(self.upstream_root, 'org/project1')
self.patch(git.Git, 'GIT_PYTHON_GIT_EXECUTABLE',

+ 7
- 4
zuul/executor/ View File

@ -764,10 +764,12 @@ class AnsibleJob(object):
# checked out
p = args['zuul']['projects'][project['canonical_name']]
p['checkout'] = selected_ref
# Delete the origin remote from each repo we set up since
# it will not be valid within the jobs.
# Set the URL of the origin remote for each repo to a bogus
# value. Keeping the remote allows tools to use it to determine
# which commits are part of the current change.
for repo in repos.values():
# This prepares each playbook and the roles needed for each.
@ -1864,7 +1866,8 @@ class ExecutorServer(object):
def _getMerger(self, root, cache_root, logger=None):
return zuul.merger.merger.Merger(
root, self.connections, self.merge_email, self.merge_name,
self.merge_speed_limit, self.merge_speed_time, cache_root, logger)
self.merge_speed_limit, self.merge_speed_time, cache_root, logger,
def start(self):
self._running = True

+ 28
- 3
zuul/merger/ View File

@ -239,7 +239,7 @@ class Repo(object):
obj = git.objects.Object.new_from_sha(repo, binsha)
git.refs.Reference.create(repo, path, obj, force=True)
def setRefs(self, refs):
def setRefs(self, refs, keep_remotes=False):
repo = self.createRepoObject()
current_refs = {}
for ref in repo.refs:
@ -248,9 +248,22 @@ class Repo(object):
for path, hexsha in refs.items():
self.setRef(path, hexsha, repo)
ref = current_refs.get(path)
if keep_remotes and ref:
for path in unseen:
self.deleteRef(path, repo)
def setRemoteRef(self, branch, rev):
repo = self.createRepoObject()
origin_ref = repo.remotes.origin.refs[branch]
except IndexError:
self.log.warning("No remote ref found for branch %s", branch)
self.log.debug("Updating remote reference %s to %s", origin_ref, rev)
origin_ref.commit = rev
def deleteRef(self, path, repo=None):
if repo is None:
repo = self.createRepoObject()
@ -365,7 +378,8 @@ class Repo(object):
class Merger(object):
def __init__(self, working_root, connections, email, username,
speed_limit, speed_time, cache_root=None, logger=None):
speed_limit, speed_time, cache_root=None, logger=None,
self.logger = logger
if logger is None:
self.log = logging.getLogger("zuul.Merger")
@ -381,6 +395,10 @@ class Merger(object):
self.speed_limit = speed_limit
self.speed_time = speed_time
self.cache_root = cache_root
# Flag to determine if the merger is used for preparing repositories
# for job execution. This flag can be used to enable executor specific
# behavior e.g. to keep the 'origin' remote intact.
self.execution_context = execution_context
def _addProject(self, hostname, project_name, url, sshkey):
repo = None
@ -471,7 +489,7 @@ class Merger(object):
self.log.debug("Restore repo state for project %s/%s",
connection_name, project_name)
repo.setRefs(project, keep_remotes=self.execution_context)
def _mergeChange(self, item, ref):
repo = self.getRepo(item['connection'], item['project'])
@ -532,6 +550,13 @@ class Merger(object):
repo_state, recent)
self.log.debug("Found base commit %s for %s" % (base, key,))
if self.execution_context:
# Set origin branch to the rev of the current (speculative) base.
# This allows tools to determine the commits that are part of a
# change by looking at origin/master..master.
repo.setRemoteRef(item['branch'], base)
# Merge the change
commit = self._mergeChange(item, base)
if not commit: