The Gatekeeper, or a project gating system
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

145 lines
5.3 KiB

# Copyright 2014 Antoine "hashar" Musso
# Copyright 2014 Wikimedia Foundation Inc.
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import git
import logging
import os
import re
import yaml
from git import GitCommandError
from zuul.lib.clonemapper import CloneMapper
from zuul.merger.merger import Repo
class Cloner(object):
log = logging.getLogger("zuul.Cloner")
def __init__(self, git_base_url, projects, workspace, zuul_branch,
zuul_ref, zuul_url, branch=None, clone_map_file=None):
self.clone_map = []
self.dests = None
self.branch = branch
self.git_url = git_base_url
self.projects = projects
self.workspace = workspace
self.zuul_branch = zuul_branch
self.zuul_ref = zuul_ref
self.zuul_url = zuul_url
if clone_map_file:
def readCloneMap(self, clone_map_file):
clone_map_file = os.path.expanduser(clone_map_file)
if not os.path.exists(clone_map_file):
raise Exception("Unable to read clone map file at %s." %
clone_map_file = open(clone_map_file)
self.clone_map = yaml.load(clone_map_file).get('clonemap')"Loaded map containing %s rules", len(self.clone_map))
return self.clone_map
def execute(self):
mapper = CloneMapper(self.clone_map, self.projects)
dests = mapper.expand(workspace=self.workspace)"Preparing %s repositories", len(dests))
for project, dest in dests.iteritems():
self.prepareRepo(project, dest)"Prepared all repositories")
def cloneUpstream(self, project, dest):
git_upstream = '%s/%s' % (self.git_url, project)"Creating repo %s from upstream %s",
project, git_upstream)
repo = Repo(
if not repo.isInitialized():
raise Exception("Error cloning %s to %s" % (git_upstream, dest))
return repo
def fetchFromZuul(self, repo, project, ref):
zuul_remote = '%s/%s' % (self.zuul_url, project)
repo.fetchFrom(zuul_remote, ref)
self.log.debug("Fetched ref %s from %s", ref, project)
return True
except (ValueError, GitCommandError):
self.log.debug("Project %s in Zuul does not have ref %s",
project, ref)
return False
def prepareRepo(self, project, dest):
"""Clone a repository for project at dest and apply a reference
suitable for testing. The reference lookup is attempted in this order:
1) Zuul reference for the indicated branch
2) Zuul reference for the master branch
3) The tip of the indicated branch
4) The tip of the master branch
repo = self.cloneUpstream(project, dest)
# Ensure that we don't have stale remotes around
override_zuul_ref = self.zuul_ref
# FIXME should be origin HEAD branch which might not be 'master'
fallback_branch = 'master'
fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
if self.branch:
override_zuul_ref = re.sub(self.zuul_branch, self.branch,
if repo.hasBranch(self.branch):
self.log.debug("upstream repo has branch %s", self.branch)
fallback_branch = self.branch
fallback_zuul_ref = self.zuul_ref
self.log.exception("upstream repo is missing branch %s",
if (self.fetchFromZuul(repo, project, override_zuul_ref)
or (fallback_zuul_ref != override_zuul_ref and
self.fetchFromZuul(repo, project, fallback_zuul_ref))
# Work around a bug in GitPython which can not parse FETCH_HEAD
gitcmd = git.Git(dest)
fetch_head = gitcmd.rev_parse('FETCH_HEAD')
repo.checkout(fetch_head)"Prepared %s repo with commit %s",
project, fetch_head)
# Checkout branch
self.log.debug("Falling back to branch %s", fallback_branch)
repo.checkout('remotes/origin/%s' % fallback_branch)
except (ValueError, GitCommandError):
self.log.exception("Fallback branch not found: %s",
fallback_branch)"Prepared %s repo with branch %s",
project, fallback_branch)