The Gatekeeper, or a project gating system
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

314 lines
11 KiB

# Copyright 2012 Hewlett-Packard Development Company, L.P.
# Copyright 2013-2014 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import git
import os
import logging
import zuul.model
class ZuulReference(git.Reference):
_common_path_default = "refs/zuul"
_points_to_commits_only = True
class Repo(object):
log = logging.getLogger("zuul.Repo")
def __init__(self, remote, local, email, username):
self.remote_url = remote
self.local_path = local
self.email = email
self.username = username
self._initialized = False
try:
self._ensure_cloned()
except:
self.log.exception("Unable to initialize repo for %s" % remote)
def _ensure_cloned(self):
repo_is_cloned = os.path.exists(self.local_path)
if self._initialized and repo_is_cloned:
return
# If the repo does not exist, clone the repo.
if not repo_is_cloned:
self.log.debug("Cloning from %s to %s" % (self.remote_url,
self.local_path))
git.Repo.clone_from(self.remote_url, self.local_path)
repo = git.Repo(self.local_path)
if self.email:
repo.config_writer().set_value('user', 'email',
self.email)
if self.username:
repo.config_writer().set_value('user', 'name',
self.username)
repo.config_writer().write()
self._initialized = True
def isInitialized(self):
return self._initialized
def createRepoObject(self):
try:
self._ensure_cloned()
repo = git.Repo(self.local_path)
except:
self.log.exception("Unable to initialize repo for %s" %
self.local_path)
return repo
def reset(self):
repo = self.createRepoObject()
self.log.debug("Resetting repository %s" % self.local_path)
self.update()
origin = repo.remotes.origin
for ref in origin.refs:
if ref.remote_head == 'HEAD':
continue
repo.create_head(ref.remote_head, ref, force=True)
# Reset to remote HEAD (usually origin/master)
repo.head.reference = origin.refs['HEAD']
repo.head.reset(index=True, working_tree=True)
repo.git.clean('-x', '-f', '-d')
def prune(self):
repo = self.createRepoObject()
origin = repo.remotes.origin
stale_refs = origin.stale_refs
if stale_refs:
self.log.debug("Pruning stale refs: %s", stale_refs)
git.refs.RemoteReference.delete(repo, *stale_refs)
def getBranchHead(self, branch):
repo = self.createRepoObject()
branch_head = repo.heads[branch]
return branch_head.commit
def hasBranch(self, branch):
repo = self.createRepoObject()
origin = repo.remotes.origin
return branch in origin.refs
def getCommitFromRef(self, refname):
repo = self.createRepoObject()
if not refname in repo.refs:
return None
ref = repo.refs[refname]
return ref.commit
def checkout(self, ref):
repo = self.createRepoObject()
self.log.debug("Checking out %s" % ref)
repo.head.reference = ref
repo.head.reset(index=True, working_tree=True)
def cherryPick(self, ref):
repo = self.createRepoObject()
self.log.debug("Cherry-picking %s" % ref)
self.fetch(ref)
repo.git.cherry_pick("FETCH_HEAD")
return repo.head.commit
def merge(self, ref, strategy=None):
repo = self.createRepoObject()
args = []
if strategy:
args += ['-s', strategy]
args.append('FETCH_HEAD')
self.fetch(ref)
self.log.debug("Merging %s with args %s" % (ref, args))
repo.git.merge(*args)
return repo.head.commit
def fetch(self, ref):
repo = self.createRepoObject()
# The git.remote.fetch method may read in git progress info and
# interpret it improperly causing an AssertionError. Because the
# data was fetched properly subsequent fetches don't seem to fail.
# So try again if an AssertionError is caught.
origin = repo.remotes.origin
try:
origin.fetch(ref)
except AssertionError:
origin.fetch(ref)
def fetchFrom(self, repository, refspec):
repo = self.createRepoObject()
repo.git.fetch(repository, refspec)
def createZuulRef(self, ref, commit='HEAD'):
repo = self.createRepoObject()
self.log.debug("CreateZuulRef %s at %s" % (ref, commit))
ref = ZuulReference.create(repo, ref, commit)
return ref.commit
def push(self, local, remote):
repo = self.createRepoObject()
self.log.debug("Pushing %s:%s to %s" % (local, remote,
self.remote_url))
repo.remotes.origin.push('%s:%s' % (local, remote))
def update(self):
repo = self.createRepoObject()
self.log.debug("Updating repository %s" % self.local_path)
origin = repo.remotes.origin
origin.update()
class Merger(object):
log = logging.getLogger("zuul.Merger")
def __init__(self, working_root, sshkey, email, username):
self.repos = {}
self.working_root = working_root
if not os.path.exists(working_root):
os.makedirs(working_root)
if sshkey:
self._makeSSHWrapper(sshkey)
self.email = email
self.username = username
def _makeSSHWrapper(self, key):
name = os.path.join(self.working_root, '.ssh_wrapper')
fd = open(name, 'w')
fd.write('#!/bin/bash\n')
fd.write('ssh -i %s $@\n' % key)
fd.close()
os.chmod(name, 0755)
os.environ['GIT_SSH'] = name
def addProject(self, project, url):
repo = None
try:
path = os.path.join(self.working_root, project)
repo = Repo(url, path, self.email, self.username)
self.repos[project] = repo
except Exception:
self.log.exception("Unable to add project %s" % project)
return repo
def getRepo(self, project, url):
if project in self.repos:
return self.repos[project]
if not url:
raise Exception("Unable to set up repo for project %s"
" without a url" % (project,))
return self.addProject(project, url)
def updateRepo(self, project, url):
repo = self.getRepo(project, url)
try:
self.log.info("Updating local repository %s", project)
repo.update()
except Exception:
self.log.exception("Unable to update %s", project)
def _mergeChange(self, item, ref):
repo = self.getRepo(item['project'], item['url'])
try:
repo.checkout(ref)
except Exception:
self.log.exception("Unable to checkout %s" % ref)
return None
try:
mode = item['merge_mode']
if mode == zuul.model.MERGER_MERGE:
commit = repo.merge(item['refspec'])
elif mode == zuul.model.MERGER_MERGE_RESOLVE:
commit = repo.merge(item['refspec'], 'resolve')
elif mode == zuul.model.MERGER_CHERRY_PICK:
commit = repo.cherryPick(item['refspec'])
else:
raise Exception("Unsupported merge mode: %s" % mode)
except git.GitCommandError:
# Log git exceptions at debug level because they are
# usually benign merge conflicts
self.log.debug("Unable to merge %s" % item, exc_info=True)
return None
except Exception:
self.log.exception("Exception while merging a change:")
return None
return commit
def _mergeItem(self, item, recent):
self.log.debug("Processing refspec %s for project %s / %s ref %s" %
(item['refspec'], item['project'], item['branch'],
item['ref']))
repo = self.getRepo(item['project'], item['url'])
key = (item['project'], item['branch'])
# See if we have a commit for this change already in this repo
zuul_ref = item['branch'] + '/' + item['ref']
commit = repo.getCommitFromRef(zuul_ref)
if commit:
self.log.debug("Found commit %s for ref %s" % (commit, zuul_ref))
# Store this as the most recent commit for this
# project-branch
recent[key] = commit
return commit
self.log.debug("Unable to find commit for ref %s" % (zuul_ref,))
# We need to merge the change
# Get the most recent commit for this project-branch
base = recent.get(key)
if not base:
# There is none, so use the branch tip
# we need to reset here in order to call getBranchHead
self.log.debug("No base commit found for %s" % (key,))
try:
repo.reset()
except Exception:
self.log.exception("Unable to reset repo %s" % repo)
return None
base = repo.getBranchHead(item['branch'])
else:
self.log.debug("Found base commit %s for %s" % (base, key,))
# Merge the change
commit = self._mergeChange(item, base)
if not commit:
return None
# Store this commit as the most recent for this project-branch
recent[key] = commit
# Set the Zuul ref for this item to point to the most recent
# commits of each project-branch
for key, mrc in recent.items():
project, branch = key
try:
repo = self.getRepo(project, None)
zuul_ref = branch + '/' + item['ref']
repo.createZuulRef(zuul_ref, mrc)
except Exception:
self.log.exception("Unable to set zuul ref %s for "
"item %s" % (zuul_ref, item))
return None
return commit
def mergeChanges(self, items):
recent = {}
commit = None
for item in items:
if item.get("number") and item.get("patchset"):
self.log.debug("Merging for change %s,%s." %
(item["number"], item["patchset"]))
elif item.get("newrev") and item.get("oldrev"):
self.log.debug("Merging for rev %s with oldrev %s." %
(item["newrev"], item["oldrev"]))
commit = self._mergeItem(item, recent)
if not commit:
return None
return commit.hexsha