# Copyright 2012 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import git import os import logging import model class ZuulReference(git.Reference): _common_path_default = "refs/zuul" _points_to_commits_only = True class Repo(object): log = logging.getLogger("zuul.Repo") def __init__(self, remote, local, email, username): self.remote_url = remote self.local_path = local self.email = email self.username = username self._initialized = False try: self._ensure_cloned() except: self.log.exception("Unable to initialize repo for %s" % remote) def _ensure_cloned(self): if self._initialized: return if not os.path.exists(self.local_path): self.log.debug("Cloning from %s to %s" % (self.remote_url, self.local_path)) git.Repo.clone_from(self.remote_url, self.local_path) self.repo = git.Repo(self.local_path) if self.email: self.repo.config_writer().set_value('user', 'email', self.email) if self.username: self.repo.config_writer().set_value('user', 'name', self.username) self.repo.config_writer().write() self._initialized = True def recreateRepoObject(self): self._ensure_cloned() self.repo = git.Repo(self.local_path) def reset(self): self._ensure_cloned() self.log.debug("Resetting repository %s" % self.local_path) self.update() origin = self.repo.remotes.origin for ref in origin.refs: if ref.remote_head == 'HEAD': continue self.repo.create_head(ref.remote_head, ref, force=True) # Reset to remote HEAD (usually origin/master) self.repo.head.reference = origin.refs['HEAD'] self.repo.head.reset(index=True, working_tree=True) self.repo.git.clean('-x', '-f', '-d') def getBranchHead(self, branch): return self.repo.heads[branch] def checkout(self, ref): self._ensure_cloned() self.log.debug("Checking out %s" % ref) self.repo.head.reference = ref self.repo.head.reset(index=True, working_tree=True) def cherryPick(self, ref): self._ensure_cloned() self.log.debug("Cherry-picking %s" % ref) self.fetch(ref) self.repo.git.cherry_pick("FETCH_HEAD") def merge(self, ref, strategy=None): self._ensure_cloned() args = [] if strategy: args += ['-s', strategy] args.append('FETCH_HEAD') self.fetch(ref) self.log.debug("Merging %s with args %s" % (ref, args)) self.repo.git.merge(*args) def fetch(self, ref): self._ensure_cloned() # The git.remote.fetch method may read in git progress info and # interpret it improperly causing an AssertionError. Because the # data was fetched properly subsequent fetches don't seem to fail. # So try again if an AssertionError is caught. origin = self.repo.remotes.origin try: origin.fetch(ref) except AssertionError: origin.fetch(ref) # If the repository is packed, and we fetch a change that is # also entirely packed, the cache may be out of date for the # same reason as reset() above. Avoid these problems by # recreating the repo object. # https://bugs.launchpad.net/zuul/+bug/1078946 self.repo = git.Repo(self.local_path) def createZuulRef(self, ref, commit='HEAD'): self._ensure_cloned() self.log.debug("CreateZuulRef %s at %s " % (ref, commit)) ref = ZuulReference.create(self.repo, ref, commit) return ref.commit def push(self, local, remote): self._ensure_cloned() self.log.debug("Pushing %s:%s to %s " % (local, remote, self.remote_url)) self.repo.remotes.origin.push('%s:%s' % (local, remote)) def update(self): self._ensure_cloned() self.log.debug("Updating repository %s" % self.local_path) origin = self.repo.remotes.origin origin.update() # If the remote repository is repacked, the repo object's # cache may be out of date. Specifically, it caches whether # to check the loose or packed DB for a given SHA. Further, # if there was no pack or lose directory to start with, the # repo object may not even have a database for it. Avoid # these problems by recreating the repo object. self.repo = git.Repo(self.local_path) class Merger(object): log = logging.getLogger("zuul.Merger") def __init__(self, trigger, working_root, push_refs, sshkey, email, username): self.trigger = trigger self.repos = {} self.working_root = working_root if not os.path.exists(working_root): os.makedirs(working_root) self.push_refs = push_refs if sshkey: self._makeSSHWrapper(sshkey) self.email = email self.username = username def _makeSSHWrapper(self, key): name = os.path.join(self.working_root, '.ssh_wrapper') fd = open(name, 'w') fd.write('#!/bin/bash\n') fd.write('ssh -i %s $@\n' % key) fd.close() os.chmod(name, 0755) os.environ['GIT_SSH'] = name def addProject(self, project, url): try: path = os.path.join(self.working_root, project.name) repo = Repo(url, path, self.email, self.username) self.repos[project] = repo except: self.log.exception("Unable to add project %s" % project) def getRepo(self, project): r = self.repos.get(project, None) r.recreateRepoObject() return r def updateRepo(self, project): repo = self.getRepo(project) try: self.log.info("Updating local repository %s", project) repo.update() except: self.log.exception("Unable to update %s", project) def _mergeChange(self, change, ref, target_ref): repo = self.getRepo(change.project) try: repo.checkout(ref) except: self.log.exception("Unable to checkout %s" % ref) return False try: mode = change.project.merge_mode if mode == model.MERGER_MERGE: repo.merge(change.refspec) elif mode == model.MERGER_MERGE_RESOLVE: repo.merge(change.refspec, 'resolve') elif mode == model.MERGER_CHERRY_PICK: repo.cherryPick(change.refspec) else: raise Exception("Unsupported merge mode: %s" % mode) except Exception: # Log exceptions at debug level because they are # usually benign merge conflicts self.log.debug("Unable to merge %s" % change, exc_info=True) return False try: # Keep track of the last commit, it's the commit that # will be passed to jenkins because it's the commit # for the triggering change zuul_ref = change.branch + '/' + target_ref commit = repo.createZuulRef(zuul_ref, 'HEAD').hexsha except: self.log.exception("Unable to set zuul ref %s for change %s" % (zuul_ref, change)) return False return commit def mergeChanges(self, items, target_ref=None): # Merge shortcuts: # if this is the only change just merge it against its branch. # elif there are changes ahead of us that are from the same project and # branch we can merge against the commit associated with that change # instead of going back up the tree. # # Shortcuts assume some external entity is checking whether or not # changes from other projects can merge. commit = False item = items[-1] sibling_filter = lambda i: (i.change.project == item.change.project and i.change.branch == item.change.branch) sibling_items = filter(sibling_filter, items) # Only current change to merge against tip of change.branch if len(sibling_items) == 1: repo = self.getRepo(item.change.project) # we need to reset here in order to call getBranchHead try: repo.reset() except: self.log.exception("Unable to reset repo %s" % repo) return False commit = self._mergeChange(item.change, repo.getBranchHead(item.change.branch), target_ref=target_ref) # Sibling changes exist. Merge current change against newest sibling. elif (len(sibling_items) >= 2 and sibling_items[-2].current_build_set.commit): last_commit = sibling_items[-2].current_build_set.commit commit = self._mergeChange(item.change, last_commit, target_ref=target_ref) # Either change did not merge or we did not need to merge as there were # previous merge conflicts. if not commit: return commit project_branches = [] for i in reversed(items): # Here we create all of the necessary zuul refs and potentially # push them back to Gerrit. if (i.change.project, i.change.branch) in project_branches: continue repo = self.getRepo(i.change.project) if (i.change.project != item.change.project or i.change.branch != item.change.branch): # Create a zuul ref for all dependent changes project # branch combinations as this is the ref that jenkins will # use to test. The ref for change has already been set so # we skip it here. try: zuul_ref = i.change.branch + '/' + target_ref repo.createZuulRef(zuul_ref, i.current_build_set.commit) except: self.log.exception("Unable to set zuul ref %s for " "change %s" % (zuul_ref, i.change)) return False if self.push_refs: # Push the results upstream to the zuul ref after # they are created. ref = 'refs/zuul/' + i.change.branch + '/' + target_ref try: repo.push(ref, ref) complete = self.trigger.waitForRefSha(i.change.project, ref) except: self.log.exception("Unable to push %s" % ref) return False if not complete: self.log.error("Ref %s did not show up in repo" % ref) return False project_branches.append((i.change.project, i.change.branch)) return commit