# Copyright 2012 Hewlett-Packard Development Company, L.P. # Copyright 2013-2014 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import git import os import logging import zuul.model class ZuulReference(git.Reference): _common_path_default = "refs/zuul" _points_to_commits_only = True class Repo(object): log = logging.getLogger("zuul.Repo") def __init__(self, remote, local, email, username): self.remote_url = remote self.local_path = local self.email = email self.username = username self._initialized = False try: self._ensure_cloned() except: self.log.exception("Unable to initialize repo for %s" % remote) def _ensure_cloned(self): repo_is_cloned = os.path.exists(self.local_path) if self._initialized and repo_is_cloned: return # If the repo does not exist, clone the repo. if not repo_is_cloned: self.log.debug("Cloning from %s to %s" % (self.remote_url, self.local_path)) git.Repo.clone_from(self.remote_url, self.local_path) repo = git.Repo(self.local_path) if self.email: repo.config_writer().set_value('user', 'email', self.email) if self.username: repo.config_writer().set_value('user', 'name', self.username) repo.config_writer().write() self._initialized = True def isInitialized(self): return self._initialized def createRepoObject(self): try: self._ensure_cloned() repo = git.Repo(self.local_path) except: self.log.exception("Unable to initialize repo for %s" % self.local_path) return repo def reset(self): repo = self.createRepoObject() self.log.debug("Resetting repository %s" % self.local_path) self.update() origin = repo.remotes.origin for ref in origin.refs: if ref.remote_head == 'HEAD': continue repo.create_head(ref.remote_head, ref, force=True) # Reset to remote HEAD (usually origin/master) repo.head.reference = origin.refs['HEAD'] repo.head.reset(index=True, working_tree=True) repo.git.clean('-x', '-f', '-d') def prune(self): repo = self.createRepoObject() origin = repo.remotes.origin stale_refs = origin.stale_refs if stale_refs: self.log.debug("Pruning stale refs: %s", stale_refs) git.refs.RemoteReference.delete(repo, *stale_refs) def getBranchHead(self, branch): repo = self.createRepoObject() branch_head = repo.heads[branch] return branch_head.commit def hasBranch(self, branch): repo = self.createRepoObject() origin = repo.remotes.origin return branch in origin.refs def getCommitFromRef(self, refname): repo = self.createRepoObject() if not refname in repo.refs: return None ref = repo.refs[refname] return ref.commit def checkout(self, ref): repo = self.createRepoObject() self.log.debug("Checking out %s" % ref) repo.head.reference = ref repo.head.reset(index=True, working_tree=True) def cherryPick(self, ref): repo = self.createRepoObject() self.log.debug("Cherry-picking %s" % ref) self.fetch(ref) repo.git.cherry_pick("FETCH_HEAD") return repo.head.commit def merge(self, ref, strategy=None): repo = self.createRepoObject() args = [] if strategy: args += ['-s', strategy] args.append('FETCH_HEAD') self.fetch(ref) self.log.debug("Merging %s with args %s" % (ref, args)) repo.git.merge(*args) return repo.head.commit def fetch(self, ref): repo = self.createRepoObject() # The git.remote.fetch method may read in git progress info and # interpret it improperly causing an AssertionError. Because the # data was fetched properly subsequent fetches don't seem to fail. # So try again if an AssertionError is caught. origin = repo.remotes.origin try: origin.fetch(ref) except AssertionError: origin.fetch(ref) def fetchFrom(self, repository, refspec): repo = self.createRepoObject() repo.git.fetch(repository, refspec) def createZuulRef(self, ref, commit='HEAD'): repo = self.createRepoObject() self.log.debug("CreateZuulRef %s at %s" % (ref, commit)) ref = ZuulReference.create(repo, ref, commit) return ref.commit def push(self, local, remote): repo = self.createRepoObject() self.log.debug("Pushing %s:%s to %s" % (local, remote, self.remote_url)) repo.remotes.origin.push('%s:%s' % (local, remote)) def update(self): repo = self.createRepoObject() self.log.debug("Updating repository %s" % self.local_path) origin = repo.remotes.origin origin.update() class Merger(object): log = logging.getLogger("zuul.Merger") def __init__(self, working_root, sshkey, email, username): self.repos = {} self.working_root = working_root if not os.path.exists(working_root): os.makedirs(working_root) if sshkey: self._makeSSHWrapper(sshkey) self.email = email self.username = username def _makeSSHWrapper(self, key): name = os.path.join(self.working_root, '.ssh_wrapper') fd = open(name, 'w') fd.write('#!/bin/bash\n') fd.write('ssh -i %s $@\n' % key) fd.close() os.chmod(name, 0755) os.environ['GIT_SSH'] = name def addProject(self, project, url): repo = None try: path = os.path.join(self.working_root, project) repo = Repo(url, path, self.email, self.username) self.repos[project] = repo except Exception: self.log.exception("Unable to add project %s" % project) return repo def getRepo(self, project, url): if project in self.repos: return self.repos[project] if not url: raise Exception("Unable to set up repo for project %s" " without a url" % (project,)) return self.addProject(project, url) def updateRepo(self, project, url): repo = self.getRepo(project, url) try: self.log.info("Updating local repository %s", project) repo.update() except Exception: self.log.exception("Unable to update %s", project) def _mergeChange(self, item, ref): repo = self.getRepo(item['project'], item['url']) try: repo.checkout(ref) except Exception: self.log.exception("Unable to checkout %s" % ref) return None try: mode = item['merge_mode'] if mode == zuul.model.MERGER_MERGE: commit = repo.merge(item['refspec']) elif mode == zuul.model.MERGER_MERGE_RESOLVE: commit = repo.merge(item['refspec'], 'resolve') elif mode == zuul.model.MERGER_CHERRY_PICK: commit = repo.cherryPick(item['refspec']) else: raise Exception("Unsupported merge mode: %s" % mode) except git.GitCommandError: # Log git exceptions at debug level because they are # usually benign merge conflicts self.log.debug("Unable to merge %s" % item, exc_info=True) return None except Exception: self.log.exception("Exception while merging a change:") return None return commit def _mergeItem(self, item, recent): self.log.debug("Processing refspec %s for project %s / %s ref %s" % (item['refspec'], item['project'], item['branch'], item['ref'])) repo = self.getRepo(item['project'], item['url']) key = (item['project'], item['branch']) # See if we have a commit for this change already in this repo zuul_ref = item['branch'] + '/' + item['ref'] commit = repo.getCommitFromRef(zuul_ref) if commit: self.log.debug("Found commit %s for ref %s" % (commit, zuul_ref)) # Store this as the most recent commit for this # project-branch recent[key] = commit return commit self.log.debug("Unable to find commit for ref %s" % (zuul_ref,)) # We need to merge the change # Get the most recent commit for this project-branch base = recent.get(key) if not base: # There is none, so use the branch tip # we need to reset here in order to call getBranchHead self.log.debug("No base commit found for %s" % (key,)) try: repo.reset() except Exception: self.log.exception("Unable to reset repo %s" % repo) return None base = repo.getBranchHead(item['branch']) else: self.log.debug("Found base commit %s for %s" % (base, key,)) # Merge the change commit = self._mergeChange(item, base) if not commit: return None # Store this commit as the most recent for this project-branch recent[key] = commit # Set the Zuul ref for this item to point to the most recent # commits of each project-branch for key, mrc in recent.items(): project, branch = key try: repo = self.getRepo(project, None) zuul_ref = branch + '/' + item['ref'] repo.createZuulRef(zuul_ref, mrc) except Exception: self.log.exception("Unable to set zuul ref %s for " "item %s" % (zuul_ref, item)) return None return commit def mergeChanges(self, items): recent = {} commit = None for item in items: if item.get("number") and item.get("patchset"): self.log.debug("Merging for change %s,%s." % (item["number"], item["patchset"])) elif item.get("newrev") and item.get("oldrev"): self.log.debug("Merging for rev %s with oldrev %s." % (item["newrev"], item["oldrev"])) commit = self._mergeItem(item, recent) if not commit: return None return commit.hexsha