From 9f16d522a976fc32d841e2d2bd92d6a4216c92c5 Mon Sep 17 00:00:00 2001 From: Sachi King Date: Wed, 16 Mar 2016 12:20:45 +1100 Subject: [PATCH] Support post jobs by supporting rev checkout Currently zuul-cloner does not support post jobs, as it does not know what to checkout. This adds the ability on a per project basis to specify a revision to be checked out. When specified zuul-cloner will successfully check out the same repo as gerrit-git-prep.sh does in post jobs. Sample usage: clonemap: - name: openstack/neutron dest: ./neu - name: openstack/requirements dest: ./reqs export ZUUL_PROJECT="openstack/neutron" export ZUUL_NEWREV="a2Fhc2Rma2FzZHNkZjhkYXM4OWZhc25pb2FzODkK" export ZUUL_BRANCH="stable/liberty" zuul-cloner -m map.yaml git://git.openstack.org $ZUUL_PROJECT \ openstack/requirements This results with openstack/neutron checked out at rev a2Fhc2 and openstack/requirements at 'heads/stable/liberty' Change-Id: Ie9b03508a44f04adfbe2696cde136439ebffb9a6 --- tests/base.py | 11 +++++++++ tests/test_cloner.py | 54 ++++++++++++++++++++++++++++++++++++++++++++ zuul/cmd/cloner.py | 8 +++++++ zuul/exceptions.py | 9 ++++++++ zuul/lib/cloner.py | 48 +++++++++++++++++++++++++++++++-------- 5 files changed, 120 insertions(+), 10 deletions(-) diff --git a/tests/base.py b/tests/base.py index 405caa0ded..7945a0b838 100755 --- a/tests/base.py +++ b/tests/base.py @@ -1132,6 +1132,17 @@ class ZuulTestCase(BaseTestCase): zuul.merger.merger.reset_repo_to_head(repo) repo.git.clean('-x', '-f', '-d') + def create_commit(self, project): + path = os.path.join(self.upstream_root, project) + repo = git.Repo(path) + repo.head.reference = repo.heads['master'] + file_name = os.path.join(path, 'README') + with open(file_name, 'a') as f: + f.write('creating fake commit\n') + repo.index.add([file_name]) + commit = repo.index.commit('Creating a fake commit') + return commit.hexsha + def ref_has_change(self, ref, change): path = os.path.join(self.git_root, change.project) repo = git.Repo(path) diff --git a/tests/test_cloner.py b/tests/test_cloner.py index 137c1570ee..e3576bdad8 100644 --- a/tests/test_cloner.py +++ b/tests/test_cloner.py @@ -566,3 +566,57 @@ class TestCloner(ZuulTestCase): self.worker.hold_jobs_in_build = False self.worker.release() self.waitUntilSettled() + + def test_post_checkout(self): + project = "org/project" + path = os.path.join(self.upstream_root, project) + repo = git.Repo(path) + repo.head.reference = repo.heads['master'] + commits = [] + for i in range(0, 3): + commits.append(self.create_commit(project)) + newRev = commits[1] + + cloner = zuul.lib.cloner.Cloner( + git_base_url=self.upstream_root, + projects=[project], + workspace=self.workspace_root, + zuul_branch=None, + zuul_ref='master', + zuul_url=self.git_root, + zuul_project=project, + zuul_newrev=newRev, + ) + cloner.execute() + repos = self.getWorkspaceRepos([project]) + cloned_sha = repos[project].rev_parse('HEAD').hexsha + self.assertEqual(newRev, cloned_sha) + + def test_post_and_master_checkout(self): + project = "org/project1" + master_project = "org/project2" + path = os.path.join(self.upstream_root, project) + repo = git.Repo(path) + repo.head.reference = repo.heads['master'] + commits = [] + for i in range(0, 3): + commits.append(self.create_commit(project)) + newRev = commits[1] + + cloner = zuul.lib.cloner.Cloner( + git_base_url=self.upstream_root, + projects=[project, master_project], + workspace=self.workspace_root, + zuul_branch=None, + zuul_ref='master', + zuul_url=self.git_root, + zuul_project=project, + zuul_newrev=newRev + ) + cloner.execute() + repos = self.getWorkspaceRepos([project, master_project]) + cloned_sha = repos[project].rev_parse('HEAD').hexsha + self.assertEqual(newRev, cloned_sha) + self.assertEqual( + repos[master_project].rev_parse('HEAD').hexsha, + repos[master_project].rev_parse('master').hexsha) diff --git a/zuul/cmd/cloner.py b/zuul/cmd/cloner.py index c616aa145f..4f8b9f474a 100755 --- a/zuul/cmd/cloner.py +++ b/zuul/cmd/cloner.py @@ -27,6 +27,8 @@ ZUUL_ENV_SUFFIXES = ( 'branch', 'ref', 'url', + 'project', + 'newrev', ) @@ -98,6 +100,10 @@ class Cloner(zuul.cmd.ZuulApp): parser.error("Specifying a Zuul ref requires a Zuul url. " "Define Zuul arguments either via environment " "variables or using options above.") + if 'zuul_newrev' in zuul_args and 'zuul_project' not in zuul_args: + parser.error("ZUUL_NEWREV has been specified without " + "ZUUL_PROJECT. Please define a ZUUL_PROJECT or do " + "not set ZUUL_NEWREV.") self.args = args @@ -145,6 +151,8 @@ class Cloner(zuul.cmd.ZuulApp): clone_map_file=self.args.clone_map_file, project_branches=project_branches, cache_dir=self.args.cache_dir, + zuul_newrev=self.args.zuul_newrev, + zuul_project=self.args.zuul_project, ) cloner.execute() diff --git a/zuul/exceptions.py b/zuul/exceptions.py index 2bd2c6b4b6..40a1e40f52 100644 --- a/zuul/exceptions.py +++ b/zuul/exceptions.py @@ -22,5 +22,14 @@ class ChangeNotFound(Exception): super(ChangeNotFound, self).__init__(message) +class RevNotFound(Exception): + def __init__(self, project, rev): + self.project = project + self.revision = rev + message = ("Failed to checkout project '%s' at revision '%s'" + % (self.project, self.revision)) + super(RevNotFound, self).__init__(message) + + class MergeFailure(Exception): pass diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py index f0235a6965..62ab9388ea 100644 --- a/zuul/lib/cloner.py +++ b/zuul/lib/cloner.py @@ -20,6 +20,7 @@ import re import yaml from git import GitCommandError +from zuul import exceptions from zuul.lib.clonemapper import CloneMapper from zuul.merger.merger import Repo @@ -29,7 +30,8 @@ class Cloner(object): def __init__(self, git_base_url, projects, workspace, zuul_branch, zuul_ref, zuul_url, branch=None, clone_map_file=None, - project_branches=None, cache_dir=None): + project_branches=None, cache_dir=None, zuul_newrev=None, + zuul_project=None): self.clone_map = [] self.dests = None @@ -43,6 +45,10 @@ class Cloner(object): self.zuul_ref = zuul_ref or '' self.zuul_url = zuul_url self.project_branches = project_branches or {} + self.project_revisions = {} + + if zuul_newrev and zuul_project: + self.project_revisions[zuul_project] = zuul_newrev if clone_map_file: self.readCloneMap(clone_map_file) @@ -119,10 +125,15 @@ class Cloner(object): """Clone a repository for project at dest and apply a reference suitable for testing. The reference lookup is attempted in this order: - 1) Zuul reference for the indicated branch - 2) Zuul reference for the master branch - 3) The tip of the indicated branch - 4) The tip of the master branch + 1) The indicated revision for specific project + 2) Zuul reference for the indicated branch + 3) Zuul reference for the master branch + 4) The tip of the indicated branch + 5) The tip of the master branch + + If an "indicated revision" is specified for this project, and we are + unable to meet this requirement, we stop attempting to check this + repo out and raise a zuul.exceptions.RevNotFound exception. The "indicated branch" is one of the following: @@ -142,6 +153,10 @@ class Cloner(object): # `git branch` is happy with. repo.reset() + indicated_revision = None + if project in self.project_revisions: + indicated_revision = self.project_revisions[project] + indicated_branch = self.branch or self.zuul_branch if project in self.project_branches: indicated_branch = self.project_branches[project] @@ -167,13 +182,26 @@ class Cloner(object): else: fallback_zuul_ref = None + # If the user has requested an explicit revision to be checked out, + # we use it above all else, and if we cannot satisfy this requirement + # we raise an error and do not attempt to continue. + if indicated_revision: + self.log.info("Attempting to check out revision %s for " + "project %s", indicated_revision, project) + try: + self.fetchFromZuul(repo, project, self.zuul_ref) + commit = repo.checkout(indicated_revision) + except (ValueError, GitCommandError): + raise exceptions.RevNotFound(project, indicated_revision) + self.log.info("Prepared '%s' repo at revision '%s'", project, + indicated_revision) # If we have a non empty zuul_ref to use, use it. Otherwise we fall # back to checking out the branch. - if ((override_zuul_ref and - self.fetchFromZuul(repo, project, override_zuul_ref)) or - (fallback_zuul_ref and - fallback_zuul_ref != override_zuul_ref and - self.fetchFromZuul(repo, project, fallback_zuul_ref))): + elif ((override_zuul_ref and + self.fetchFromZuul(repo, project, override_zuul_ref)) or + (fallback_zuul_ref and + fallback_zuul_ref != override_zuul_ref and + self.fetchFromZuul(repo, project, fallback_zuul_ref))): # Work around a bug in GitPython which can not parse FETCH_HEAD gitcmd = git.Git(dest) fetch_head = gitcmd.rev_parse('FETCH_HEAD')