diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst index 4e1c33dc59..2220c4ecb7 100644 --- a/doc/source/user/jobs.rst +++ b/doc/source/user/jobs.rst @@ -45,8 +45,13 @@ fallback or default branch will be used). If your job needs to operate on multiple branches, simply checkout the appropriate branches of these git repos to ensure that the job results reflect the proposed future state that Zuul is testing, and all dependencies are present. -Do not use any git remotes; the local repositories are guaranteed to -be up to date. + +The git repositories will have a remote ``origin`` with refs pointing +to the previous change in the speculative state. This means that e.g. +a ``git diff origin/..`` will show the changes being +tested. Note that the ``origin`` URL is set to a bogus value +(``file:///dev/null``) and can not be used for updating the repository +state; the local repositories are guaranteed to be up to date. The repositories will be placed on the filesystem in directories corresponding with the canonical hostname of their source connection. diff --git a/releasenotes/notes/git-remote-refs-71bd2fc2bb05155d.yaml b/releasenotes/notes/git-remote-refs-71bd2fc2bb05155d.yaml new file mode 100644 index 0000000000..3c570c8e07 --- /dev/null +++ b/releasenotes/notes/git-remote-refs-71bd2fc2bb05155d.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Git repositories will have a ``origin`` remote with refs pointing to the + previous change in the speculative state. + + This allows jobs to determine the commits that are part of a change, which + was not possible before. The remote URL is set to a bogus value which + won't work with git commands that need to talk to the remote repository. diff --git a/tests/unit/test_executor.py b/tests/unit/test_executor.py index bed2895769..22641ecfed 100755 --- a/tests/unit/test_executor.py +++ b/tests/unit/test_executor.py @@ -41,6 +41,9 @@ class TestExecutorRepos(ZuulTestCase): 'Project %s commit for build %s #%s should ' 'be on the correct branch' % ( project, build, number)) + # Remote 'origin' needs to be kept intact with a bogus URL + self.assertEqual(repo.remotes.origin.url, 'file:///dev/null') + self.assertIn(state['branch'], repo.remotes.origin.refs) if 'commit' in state: self.assertEqual(state['commit'], str(repo.commit('HEAD')), diff --git a/tests/unit/test_merger_repo.py b/tests/unit/test_merger_repo.py index 984644fc56..8620d1c671 100644 --- a/tests/unit/test_merger_repo.py +++ b/tests/unit/test_merger_repo.py @@ -76,6 +76,39 @@ class TestMergerRepo(ZuulTestCase): sub_repo.createRepoObject().remotes[0].url, message="Sub repository points to upstream project2") + def test_set_refs(self): + parent_path = os.path.join(self.upstream_root, 'org/project1') + remote_sha = self.create_commit('org/project1') + self.create_branch('org/project1', 'foobar') + + work_repo = Repo(parent_path, self.workspace_root, + 'none@example.org', 'User Name', '0', '0') + repo = git.Repo(self.workspace_root) + new_sha = repo.heads.foobar.commit.hexsha + + work_repo.setRefs({'refs/heads/master': new_sha}, True) + self.assertEqual(work_repo.getBranchHead('master').hexsha, new_sha) + self.assertIn('master', repo.remotes.origin.refs) + + work_repo.setRefs({'refs/heads/master': remote_sha}) + self.assertEqual(work_repo.getBranchHead('master').hexsha, remote_sha) + self.assertNotIn('master', repo.remotes.origin.refs) + + def test_set_remote_ref(self): + parent_path = os.path.join(self.upstream_root, 'org/project1') + commit_sha = self.create_commit('org/project1') + self.create_commit('org/project1') + + work_repo = Repo(parent_path, self.workspace_root, + 'none@example.org', 'User Name', '0', '0') + work_repo.setRemoteRef('master', commit_sha) + work_repo.setRemoteRef('invalid', commit_sha) + + repo = git.Repo(self.workspace_root) + self.assertEqual(repo.remotes.origin.refs.master.commit.hexsha, + commit_sha) + self.assertNotIn('invalid', repo.remotes.origin.refs) + def test_clone_timeout(self): parent_path = os.path.join(self.upstream_root, 'org/project1') self.patch(git.Git, 'GIT_PYTHON_GIT_EXECUTABLE', diff --git a/zuul/executor/server.py b/zuul/executor/server.py index d292d76b0f..e3ba32d3b4 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -764,10 +764,12 @@ class AnsibleJob(object): # checked out p = args['zuul']['projects'][project['canonical_name']] p['checkout'] = selected_ref - # Delete the origin remote from each repo we set up since - # it will not be valid within the jobs. + + # Set the URL of the origin remote for each repo to a bogus + # value. Keeping the remote allows tools to use it to determine + # which commits are part of the current change. for repo in repos.values(): - repo.deleteRemote('origin') + repo.setRemoteUrl('file:///dev/null') # This prepares each playbook and the roles needed for each. self.preparePlaybooks(args) @@ -1864,7 +1866,8 @@ class ExecutorServer(object): def _getMerger(self, root, cache_root, logger=None): return zuul.merger.merger.Merger( root, self.connections, self.merge_email, self.merge_name, - self.merge_speed_limit, self.merge_speed_time, cache_root, logger) + self.merge_speed_limit, self.merge_speed_time, cache_root, logger, + execution_context=True) def start(self): self._running = True diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py index aba8645c0a..ccaaf5bfcd 100644 --- a/zuul/merger/merger.py +++ b/zuul/merger/merger.py @@ -239,7 +239,7 @@ class Repo(object): obj = git.objects.Object.new_from_sha(repo, binsha) git.refs.Reference.create(repo, path, obj, force=True) - def setRefs(self, refs): + def setRefs(self, refs, keep_remotes=False): repo = self.createRepoObject() current_refs = {} for ref in repo.refs: @@ -248,9 +248,22 @@ class Repo(object): for path, hexsha in refs.items(): self.setRef(path, hexsha, repo) unseen.discard(path) + ref = current_refs.get(path) + if keep_remotes and ref: + unseen.discard('refs/remotes/origin/{}'.format(ref.name)) for path in unseen: self.deleteRef(path, repo) + def setRemoteRef(self, branch, rev): + repo = self.createRepoObject() + try: + origin_ref = repo.remotes.origin.refs[branch] + except IndexError: + self.log.warning("No remote ref found for branch %s", branch) + return + self.log.debug("Updating remote reference %s to %s", origin_ref, rev) + origin_ref.commit = rev + def deleteRef(self, path, repo=None): if repo is None: repo = self.createRepoObject() @@ -365,7 +378,8 @@ class Repo(object): class Merger(object): def __init__(self, working_root, connections, email, username, - speed_limit, speed_time, cache_root=None, logger=None): + speed_limit, speed_time, cache_root=None, logger=None, + execution_context=False): self.logger = logger if logger is None: self.log = logging.getLogger("zuul.Merger") @@ -381,6 +395,10 @@ class Merger(object): self.speed_limit = speed_limit self.speed_time = speed_time self.cache_root = cache_root + # Flag to determine if the merger is used for preparing repositories + # for job execution. This flag can be used to enable executor specific + # behavior e.g. to keep the 'origin' remote intact. + self.execution_context = execution_context def _addProject(self, hostname, project_name, url, sshkey): repo = None @@ -471,7 +489,7 @@ class Merger(object): return self.log.debug("Restore repo state for project %s/%s", connection_name, project_name) - repo.setRefs(project) + repo.setRefs(project, keep_remotes=self.execution_context) def _mergeChange(self, item, ref): repo = self.getRepo(item['connection'], item['project']) @@ -532,6 +550,13 @@ class Merger(object): repo_state, recent) else: self.log.debug("Found base commit %s for %s" % (base, key,)) + + if self.execution_context: + # Set origin branch to the rev of the current (speculative) base. + # This allows tools to determine the commits that are part of a + # change by looking at origin/master..master. + repo.setRemoteRef(item['branch'], base) + # Merge the change commit = self._mergeChange(item, base) if not commit: