zuul/zuul
Code Issues Proposed changes

Support post jobs by supporting rev checkout

Browse Source 
Currently zuul-cloner does not support post jobs, as it does not know
what to checkout.  This adds the ability on a per project basis to
specify a revision to be checked out.  When specified zuul-cloner
will successfully check out the same repo as gerrit-git-prep.sh does
in post jobs.

Sample usage:
clonemap:
  - name: openstack/neutron
    dest: ./neu
  - name: openstack/requirements
    dest: ./reqs

export ZUUL_PROJECT="openstack/neutron"
export ZUUL_NEWREV="a2Fhc2Rma2FzZHNkZjhkYXM4OWZhc25pb2FzODkK"
export ZUUL_BRANCH="stable/liberty"

zuul-cloner -m map.yaml git://git.openstack.org $ZUUL_PROJECT \
openstack/requirements

This results with openstack/neutron checked out at rev a2Fhc2 and
openstack/requirements at 'heads/stable/liberty'

Change-Id: Ie9b03508a44f04adfbe2696cde136439ebffb9a6
This commit is contained in:
Sachi King 2016-03-16 12:20:45 +11:00
parent 7c21047824
commit 9f16d522a9
5 changed files with 120 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tests/base.py
View File

@ -1132,6 +1132,17 @@ class ZuulTestCase(BaseTestCase):
zuul.merger.merger.reset_repo_to_head(repo)
repo.git.clean('-x', '-f', '-d')
def create_commit(self, project):
path = os.path.join(self.upstream_root, project)
repo = git.Repo(path)
repo.head.reference = repo.heads['master']
file_name = os.path.join(path, 'README')
with open(file_name, 'a') as f:
f.write('creating fake commit\n')
repo.index.add([file_name])
commit = repo.index.commit('Creating a fake commit')
return commit.hexsha
def ref_has_change(self, ref, change):
path = os.path.join(self.git_root, change.project)
repo = git.Repo(path)

54
tests/test_cloner.py
View File

@ -566,3 +566,57 @@ class TestCloner(ZuulTestCase):
self.worker.hold_jobs_in_build = False
self.worker.release()
self.waitUntilSettled()
def test_post_checkout(self):
project = "org/project"
path = os.path.join(self.upstream_root, project)
repo = git.Repo(path)
repo.head.reference = repo.heads['master']
commits = []
for i in range(0, 3):
commits.append(self.create_commit(project))
newRev = commits[1]
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.upstream_root,
projects=[project],
workspace=self.workspace_root,
zuul_branch=None,
zuul_ref='master',
zuul_url=self.git_root,
zuul_project=project,
zuul_newrev=newRev,
)
cloner.execute()
repos = self.getWorkspaceRepos([project])
cloned_sha = repos[project].rev_parse('HEAD').hexsha
self.assertEqual(newRev, cloned_sha)
def test_post_and_master_checkout(self):
project = "org/project1"
master_project = "org/project2"
path = os.path.join(self.upstream_root, project)
repo = git.Repo(path)
repo.head.reference = repo.heads['master']
commits = []
for i in range(0, 3):
commits.append(self.create_commit(project))
newRev = commits[1]
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.upstream_root,
projects=[project, master_project],
workspace=self.workspace_root,
zuul_branch=None,
zuul_ref='master',
zuul_url=self.git_root,
zuul_project=project,
zuul_newrev=newRev
)
cloner.execute()
repos = self.getWorkspaceRepos([project, master_project])
cloned_sha = repos[project].rev_parse('HEAD').hexsha
self.assertEqual(newRev, cloned_sha)
self.assertEqual(
repos[master_project].rev_parse('HEAD').hexsha,
repos[master_project].rev_parse('master').hexsha)

8
zuul/cmd/cloner.py
View File

@ -27,6 +27,8 @@ ZUUL_ENV_SUFFIXES = (
'branch',
'ref',
'url',
'project',
'newrev',
)
@ -98,6 +100,10 @@ class Cloner(zuul.cmd.ZuulApp):
parser.error("Specifying a Zuul ref requires a Zuul url. "
"Define Zuul arguments either via environment "
"variables or using options above.")
if 'zuul_newrev' in zuul_args and 'zuul_project' not in zuul_args:
parser.error("ZUUL_NEWREV has been specified without "
"ZUUL_PROJECT. Please define a ZUUL_PROJECT or do "
"not set ZUUL_NEWREV.")
self.args = args
@ -145,6 +151,8 @@ class Cloner(zuul.cmd.ZuulApp):
clone_map_file=self.args.clone_map_file,
project_branches=project_branches,
cache_dir=self.args.cache_dir,
zuul_newrev=self.args.zuul_newrev,
zuul_project=self.args.zuul_project,
)
cloner.execute()

9
zuul/exceptions.py
View File

@ -22,5 +22,14 @@ class ChangeNotFound(Exception):
super(ChangeNotFound, self).__init__(message)
class RevNotFound(Exception):
def __init__(self, project, rev):
self.project = project
self.revision = rev
message = ("Failed to checkout project '%s' at revision '%s'"
% (self.project, self.revision))
super(RevNotFound, self).__init__(message)
class MergeFailure(Exception):
pass

48
zuul/lib/cloner.py
View File

@ -20,6 +20,7 @@ import re
import yaml
from git import GitCommandError
from zuul import exceptions
from zuul.lib.clonemapper import CloneMapper
from zuul.merger.merger import Repo
@ -29,7 +30,8 @@ class Cloner(object):
def __init__(self, git_base_url, projects, workspace, zuul_branch,
zuul_ref, zuul_url, branch=None, clone_map_file=None,
project_branches=None, cache_dir=None):
project_branches=None, cache_dir=None, zuul_newrev=None,
zuul_project=None):
self.clone_map = []
self.dests = None
@ -43,6 +45,10 @@ class Cloner(object):
self.zuul_ref = zuul_ref or ''
self.zuul_url = zuul_url
self.project_branches = project_branches or {}
self.project_revisions = {}
if zuul_newrev and zuul_project:
self.project_revisions[zuul_project] = zuul_newrev
if clone_map_file:
self.readCloneMap(clone_map_file)
@ -119,10 +125,15 @@ class Cloner(object):
"""Clone a repository for project at dest and apply a reference
suitable for testing. The reference lookup is attempted in this order:
1) Zuul reference for the indicated branch
2) Zuul reference for the master branch
3) The tip of the indicated branch
4) The tip of the master branch
1) The indicated revision for specific project
2) Zuul reference for the indicated branch
3) Zuul reference for the master branch
4) The tip of the indicated branch
5) The tip of the master branch
If an "indicated revision" is specified for this project, and we are
unable to meet this requirement, we stop attempting to check this
repo out and raise a zuul.exceptions.RevNotFound exception.
The "indicated branch" is one of the following:
@ -142,6 +153,10 @@ class Cloner(object):
# `git branch` is happy with.
repo.reset()
indicated_revision = None
if project in self.project_revisions:
indicated_revision = self.project_revisions[project]
indicated_branch = self.branch or self.zuul_branch
if project in self.project_branches:
indicated_branch = self.project_branches[project]
@ -167,13 +182,26 @@ class Cloner(object):
else:
fallback_zuul_ref = None
# If the user has requested an explicit revision to be checked out,
# we use it above all else, and if we cannot satisfy this requirement
# we raise an error and do not attempt to continue.
if indicated_revision:
self.log.info("Attempting to check out revision %s for "
"project %s", indicated_revision, project)
try:
self.fetchFromZuul(repo, project, self.zuul_ref)
commit = repo.checkout(indicated_revision)
except (ValueError, GitCommandError):
raise exceptions.RevNotFound(project, indicated_revision)
self.log.info("Prepared '%s' repo at revision '%s'", project,
indicated_revision)
# If we have a non empty zuul_ref to use, use it. Otherwise we fall
# back to checking out the branch.
if ((override_zuul_ref and
self.fetchFromZuul(repo, project, override_zuul_ref)) or
(fallback_zuul_ref and
fallback_zuul_ref != override_zuul_ref and
self.fetchFromZuul(repo, project, fallback_zuul_ref))):
elif ((override_zuul_ref and
self.fetchFromZuul(repo, project, override_zuul_ref)) or
(fallback_zuul_ref and
fallback_zuul_ref != override_zuul_ref and
self.fetchFromZuul(repo, project, fallback_zuul_ref))):
# Work around a bug in GitPython which can not parse FETCH_HEAD
gitcmd = git.Git(dest)
fetch_head = gitcmd.rev_parse('FETCH_HEAD')