The Gatekeeper, or a project gating system
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

171 lines
6.5 KiB

# Copyright 2014 Antoine "hashar" Musso
# Copyright 2014 Wikimedia Foundation Inc.
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import git
import logging
import os
import re
import yaml
from git import GitCommandError
from zuul.lib.clonemapper import CloneMapper
from zuul.merger.merger import Repo
class Cloner(object):
log = logging.getLogger("zuul.Cloner")
def __init__(self, git_base_url, projects, workspace, zuul_branch,
zuul_ref, zuul_url, branch=None, clone_map_file=None,
project_branches=None, cache_dir=None):
self.clone_map = []
self.dests = None
self.branch = branch
self.git_url = git_base_url
self.cache_dir = cache_dir
self.projects = projects
self.workspace = workspace
self.zuul_branch = zuul_branch
self.zuul_ref = zuul_ref
self.zuul_url = zuul_url
self.project_branches = project_branches or {}
if clone_map_file:
def readCloneMap(self, clone_map_file):
clone_map_file = os.path.expanduser(clone_map_file)
if not os.path.exists(clone_map_file):
raise Exception("Unable to read clone map file at %s." %
clone_map_file = open(clone_map_file)
self.clone_map = yaml.load(clone_map_file).get('clonemap')"Loaded map containing %s rules", len(self.clone_map))
return self.clone_map
def execute(self):
mapper = CloneMapper(self.clone_map, self.projects)
dests = mapper.expand(workspace=self.workspace)"Preparing %s repositories", len(dests))
for project, dest in dests.iteritems():
self.prepareRepo(project, dest)"Prepared all repositories")
def cloneUpstream(self, project, dest):
# Check for a cached git repo first
git_cache = '%s/%s' % (self.cache_dir, project)
git_upstream = '%s/%s' % (self.git_url, project)
if (self.cache_dir and
os.path.exists(git_cache) and
not os.path.exists(dest)):
# file:// tells git not to hard-link across repos
git_cache = 'file://%s' % git_cache"Creating repo %s from cache %s",
project, git_cache)
new_repo = git.Repo.clone_from(git_cache, dest)"Updating origin remote in repo %s to %s",
project, git_upstream)
origin = new_repo.remotes.origin.config_writer.set(
'url', git_upstream)
else:"Creating repo %s from upstream %s",
project, git_upstream)
repo = Repo(
if not repo.isInitialized():
raise Exception("Error cloning %s to %s" % (git_upstream, dest))
return repo
def fetchFromZuul(self, repo, project, ref):
zuul_remote = '%s/%s' % (self.zuul_url, project)
repo.fetchFrom(zuul_remote, ref)
self.log.debug("Fetched ref %s from %s", ref, project)
return True
except (ValueError, GitCommandError):
self.log.debug("Project %s in Zuul does not have ref %s",
project, ref)
return False
def prepareRepo(self, project, dest):
"""Clone a repository for project at dest and apply a reference
suitable for testing. The reference lookup is attempted in this order:
1) Zuul reference for the indicated branch
2) Zuul reference for the master branch
3) The tip of the indicated branch
4) The tip of the master branch
The "indicated branch" is one of the following:
A) The project-specific override branch (from project_branches arg)
B) The user specified branch (from the branch arg)
C) ZUUL_BRANCH (from the zuul_branch arg)
repo = self.cloneUpstream(project, dest)
# Ensure that we don't have stale remotes around
indicated_branch = self.branch or self.zuul_branch
if project in self.project_branches:
indicated_branch = self.project_branches[project]
override_zuul_ref = re.sub(self.zuul_branch, indicated_branch,
if repo.hasBranch(indicated_branch):
self.log.debug("upstream repo has branch %s", indicated_branch)
fallback_branch = indicated_branch
self.log.debug("upstream repo is missing branch %s",
# FIXME should be origin HEAD branch which might not be 'master'
fallback_branch = 'master'
fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
if (self.fetchFromZuul(repo, project, override_zuul_ref)
or (fallback_zuul_ref != override_zuul_ref and
self.fetchFromZuul(repo, project, fallback_zuul_ref))
# Work around a bug in GitPython which can not parse FETCH_HEAD
gitcmd = git.Git(dest)
fetch_head = gitcmd.rev_parse('FETCH_HEAD')
repo.checkout(fetch_head)"Prepared %s repo with commit %s",
project, fetch_head)
# Checkout branch
self.log.debug("Falling back to branch %s", fallback_branch)
repo.checkout('remotes/origin/%s' % fallback_branch)
except (ValueError, GitCommandError):
self.log.exception("Fallback branch not found: %s",
fallback_branch)"Prepared %s repo with branch %s",
project, fallback_branch)