x/git-upstream
Code Issues Proposed changes

Re-factor and split code

Browse Source 
Change-Id: I546433079a2b16c46d0e43f81fe9a1508241947c
This commit is contained in:
Darragh Bailey
2015-07-27 02:06:44 +01:00
committed by Darragh Bailey
parent ca9eefdf12
commit 3fb410c9b6
9 changed files with 850 additions and 769 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
git_upstream/commands/drop.py
View File

@@ -15,113 +15,10 @@
# limitations under the License.
#
import re
from git import BadObject
from git_upstream.commands import GitUpstreamCommand
from git_upstream.errors import GitUpstreamError
from git_upstream.lib.utils import GitMixin
from git_upstream.lib.drop import Drop
from git_upstream.log import LogDedentMixin
try:
from git import BadName
except ImportError:
BadName = None
class DropError(GitUpstreamError):
"""Exception thrown by L{Drop}"""
pass
class Drop(LogDedentMixin, GitMixin):
"""Mark a commit to be dropped on next import.
Mark a commit as to be dropped.
The mark is applied by means of a note in upstream-merge namespace
(refs/notes/upstream-merge).
The note will contain the following header:
Dropped: Walter White <heisenberg@hp.com>
"""
DROP_HEADER = 'Dropped:'
NOTE_REF = 'refs/notes/upstream-merge'
def __init__(self, git_object=None, author=None, *args, **kwargs):
# make sure to correctly initialize inherited objects before performing
# any computation
super(Drop, self).__init__(*args, **kwargs)
# test parameters
if not git_object:
raise DropError("Commit should be provided")
try:
# test commit "id" presence
self._commit = self.repo.commit(git_object)
except (BadName, BadObject):
raise DropError(
"Commit '%s' not found (or ambiguous)" % git_object)
if not author:
self._author = '%s <%s>' % (self.repo.git.config('user.name'),
self.repo.git.config('user.email'))
else:
self._author = author
# test that we can use this git repo
if not self.is_detached():
raise DropError("In 'detached HEAD' state")
# To Do: check if it possible and useful.
if self.repo.bare:
raise DropError("Cannot add notes in bare repositories")
@property
def commit(self):
"""Commit to be marked as dropped."""
return self._commit
@property
def author(self):
"""Commit to be marked as dropped."""
return self._author
def check_duplicates(self):
"""Check if a dropped header is already present"""
note = self.commit.note(note_ref=Drop.NOTE_REF)
if note:
pattern = '^%s\s*(.+)' % Drop.DROP_HEADER
m = re.search(pattern, note, re.MULTILINE | re.IGNORECASE)
if m:
self.log.warning(
"""Drop header already present in the note for commit '%s':
%s""" % (self.commit, m.group(1)))
return False
return True
def mark(self):
"""
Create the note for the given commit with the given change-ids.
"""
self.log.debug("Creating a note for commit '%s'", self.commit)
if self.check_duplicates():
git_note = '%s %s\n' % (Drop.DROP_HEADER, self.author)
self.log.debug('With the following content:')
self.log.debug(git_note)
self.commit.append_note(git_note, note_ref=Drop.NOTE_REF)
else:
self.log.warning(
"Drop note has not been added as '%s' already has one" %
self.commit)
class DropCommand(LogDedentMixin, GitUpstreamCommand):
"""Mark a commit as dropped.

502
git_upstream/commands/import.py
View File

@@ -15,508 +15,14 @@
# limitations under the License.
#
from abc import ABCMeta
from abc import abstractmethod
from collections import Sequence
from git import GitCommandError
from git_upstream.commands import GitUpstreamCommand
from git_upstream.errors import GitUpstreamError
from git_upstream.lib.rebaseeditor import RebaseEditor
from git_upstream.lib.searchers import DiscardDuplicateGerritChangeId
from git_upstream.lib.searchers import DroppedCommitFilter
from git_upstream.lib.searchers import NoMergeCommitFilter
from git_upstream.lib.searchers import ReverseCommitFilter
from git_upstream.lib.searchers import SupersededCommitFilter
from git_upstream.lib.searchers import UpstreamMergeBaseSearcher
from git_upstream.lib.utils import GitMixin
from git_upstream.lib.importupstream import ImportUpstream
from git_upstream.lib.importupstream import ImportUpstreamError
from git_upstream.lib.strategies import ImportStrategiesFactory
from git_upstream.lib.strategies import LocateChangesWalk
from git_upstream.log import LogDedentMixin
class ImportUpstreamError(GitUpstreamError):
"""Exception thrown by L{ImportUpstream}"""
pass
class ImportUpstream(LogDedentMixin, GitMixin):
"""Import code from an upstream project and merge in additional branches
to create a new branch unto which changes that are not upstream but are
on the local branch are applied.
"""
def __init__(self, branch=None, upstream=None, import_branch=None,
extra_branches=None, *args, **kwargs):
if not extra_branches:
extra_branches = []
self._branch = branch
self._upstream = upstream
self._import_branch = import_branch
self._extra_branches = extra_branches
# make sure to correctly initialise inherited objects before performing
# any computation
super(ImportUpstream, self).__init__(*args, **kwargs)
# test that we can use this git repo
if not self.is_detached():
raise ImportUpstreamError("In 'detached HEAD' state")
if self.repo.bare:
raise ImportUpstreamError("Cannot perform imports in bare repos")
if self.branch == 'HEAD':
self._branch = str(self.repo.active_branch)
# validate branches exist and log all failures
branches = [
self.branch,
self.upstream
]
branches.extend(self.extra_branches)
invalid_ref = False
for branch in branches:
if not any(head for head in self.repo.heads
if head.name == branch):
msg = "Specified ref does not exist: '%s'"
self.log.error(msg, branch)
invalid_ref = True
if invalid_ref:
raise ImportUpstreamError("Invalid ref")
@property
def branch(self):
"""Branch to search for branch changes to apply when importing."""
return self._branch
@property
def upstream(self):
"""Branch containing the upstream project code base to track."""
return self._upstream
@property
def import_branch(self):
"""Pattern to use to generate the name, or user specified branch name
to use for import.
"""
return self._import_branch
@property
def extra_branches(self):
"""Branch containing the additional branches to be merged with the
upstream when importing.
"""
return self._extra_branches
def _set_branch(self, branch, commit, checkout=False, force=False):
if str(self.repo.active_branch) == branch:
self.log.info(
"""\
Resetting branch '%s' to specified commit '%s'
git reset --hard %s
""", branch, commit, commit)
self.git.reset(commit, hard=True)
elif checkout:
if force:
checkout_opt = '-B'
else:
checkout_opt = '-b'
self.log.info(
"""\
Checking out branch '%s' using specified commit '%s'
git checkout %s %s %s
""", branch, commit, checkout_opt, branch, commit)
self.git.checkout(checkout_opt, branch, commit)
else:
self.log.info(
"""\
Creating branch '%s' from specified commit '%s'
git branch --force %s %s
""", branch, commit, branch, commit)
self.git.branch(branch, commit, force=force)
def create_import(self, commit=None, import_branch=None, checkout=False,
force=False):
"""Create the import branch from the specified commit.
If the branch already exists abort if force is false
If current branch, reset the head to the specified commit
If checkout is true, switch and reset the branch to the commit
Otherwise just reset the branch to the specified commit
If the branch doesn't exist, create it and switch to it
automatically if checkout is true.
"""
if not commit:
commit = self.upstream
try:
self.git.show_ref(commit, quiet=True, heads=True)
except GitCommandError as e:
msg = "Invalid commit '%s' specified to import from"
self.log.error(msg, commit)
raise ImportUpstreamError((msg + ": %s"), commit, e)
if not import_branch:
import_branch = self.import_branch
# use describe in order to be certain about unique identifying 'commit'
# Create a describe string with the following format:
# <describe upstream>[-<extra branch abbref hash>]*
#
# Simply appends the 7 character ref abbreviation for each extra branch
# prefixed with '-', for each extra branch in the order they are given.
describe_commit = self.git.describe(commit, tags=True,
with_exceptions=False)
if not describe_commit:
self.log.warning("No tag describes the upstream branch")
describe_commit = self.git.describe(commit, always=True, tags=True)
self.log.info("""\
Using '%s' to describe:
%s
""", describe_commit, commit)
describe_branches = [describe_commit]
describe_branches.extend([self.git.rev_parse(b, short=True)
for b in self.extra_branches])
import_describe = "-".join(describe_branches)
self._import_branch = self.import_branch.format(
describe=import_describe)
self._import_branch = import_branch.format(describe=import_describe)
base = self._import_branch + "-base"
self.log.debug("Creating and switching to import branch base '%s' "
"created from '%s' (%s)", base, self.upstream, commit)
self.log.info(
"""\
Checking if import branch '%s' already exists:
git branch --list %s
""", base, base)
if self.git.show_ref("refs/heads/" + base, verify=True,
with_exceptions=False) and not force:
msg = "Import branch '%s' already exists, set 'force' to replace"
self.log.error(msg, self.import_branch)
raise ImportUpstreamError(msg % self.import_branch)
self._set_branch(base, commit, checkout, force)
if self.extra_branches:
self.log.info(
"""\
Merging additional branch(es) '%s' into import branch '%s'
git checkout %s
git merge %s
""", ", ".join(self.extra_branches), base, base,
" ".join(self.extra_branches))
self.git.checkout(base)
self.git.merge(*self.extra_branches)
def _linearise(self, branch, sequence, previous_import):
counter = len(sequence) - 1
ancestors = set()
self._set_branch(branch, previous_import, checkout=True, force=True)
root = previous_import.hexsha
while counter > 0:
# add commit to list of ancestors to check
ancestors.add(root)
# look for merge commits that are not part of ancestry path
for idx in xrange(counter - 1, -1, -1):
commit = sequence[idx]
# if there is only one parent, no need to check the others
if len(commit.parents) < 2:
ancestors.add(commit.hexsha)
elif any(p.hexsha not in ancestors for p in commit.parents):
self.log.debug("Rebase upto commit SHA1: %s",
commit.hexsha)
idx = idx + 1
break
else:
ancestors.add(commit.hexsha)
tip = sequence[idx].hexsha
self.log.info("Rebasing from %s to %s", root, tip)
previous = self.git.rev_parse(branch)
self.log.info("Rebasing onto '%s'", previous)
if root == previous and idx == 0:
# special case, we are already linear
self.log.info("Already in a linear layout")
return
self._set_branch(branch, tip, force=True)
try:
self.log.debug(
"""\
git rebase -p --onto=%s \\
%s %s
""", previous, root, branch)
self.git.rebase(root, branch, onto=previous, p=True)
except Exception:
self.git.rebase(abort=True, with_exceptions=False)
raise
counter = idx - 1
# set root commit for next loop
root = sequence[counter].hexsha
def apply(self, strategy, interactive=False):
"""Apply list of commits given onto latest import of upstream"""
commit_list = list(strategy.filtered_iter())
if len(commit_list) == 0:
self.log.notice("There are no local changes to be applied!")
return False
self.log.debug(
"""\
Should apply the following list of commits
%s
""", "\n ".join([c.hexsha for c in commit_list]))
base = self.import_branch + "-base"
self._set_branch(self.import_branch, self.branch, force=True)
self.log.info(
"""\
Creating import branch '%s' from specified commit '%s' in prep to
linearize the local changes before transposing to the new upstream:
git branch --force %s %s
""", self.import_branch, self.branch, self.import_branch,
self.branch)
self.log.notice("Attempting to linearise previous changes")
# attempt to silently linearize the current carried changes as a branch
# based on the previous located import commit. This provides a sane
# abort result for if the user needs to abort the rebase of this branch
# onto the new point upstream that was requested to import from.
try:
self._linearise(self.import_branch, strategy,
strategy.searcher.commit)
except Exception:
# Could ask user if they want to try and use the non clean route
# provided they don't mind that 'git rebase --abort' will result
# in a virtually useless local import branch
self.log.warning(
"""\
Exception occurred during linearisation of local changes on to
previous import to simplify behaviour should user need to abort
the rebase that applies these changes to the latest import
point. Attempting to tidy up state.
Do not Ctrl+C unless you wish to need to clean up your git
repository by hand.
""")
# reset head back to the tip of the changes to be rebased
self._set_branch(self.import_branch, self.branch, force=True)
rebase = RebaseEditor(interactive, repo=self.repo)
if len(commit_list):
first = commit_list[0]
self.log.info(
"""\
Rebase changes, dropping merges through editor:
git rebase --onto %s \\
%s %s
""", base, first.parents[0].hexsha, self.import_branch)
status, out, err = rebase.run(commit_list,
first.parents[0].hexsha,
self.import_branch,
onto=base)
if status:
if err and err.startswith("Nothing to do"):
# cancelled by user
self.log.notice("Cancelled by user")
return False
self.log.error("Rebase failed, will need user intervention to "
"resolve.")
if out:
self.log.notice(out)
if err:
self.log.notice(err)
# once we support resuming/finishing add a message here to tell
# the user to rerun this tool with the appropriate options to
# complete
return False
self.log.notice("Successfully applied all locally carried changes")
else:
self.log.warning("Warning, nothing to do: locally carried " +
"changes already rebased onto " + self.upstream)
return True
def resume(self, args):
"""Resume previous partial import"""
raise NotImplementedError
def finish(self):
"""Finish import
Finish the import by merging the import branch to the target while
performing suitable verification checks.
"""
self.log.info("No verification checks enabled")
self.git.checkout(self.branch)
current_sha = self.git.rev_parse("HEAD")
try:
self.log.info(
"""\
Merging by inverting the 'ours' strategy discard all changes
and replace existing branch contents with the new import.
""")
self.log.info(
"""\
Merging import branch to HEAD and ignoring changes:
git merge -s ours --no-commit %s
""", self.import_branch)
self.git.merge('-s', 'ours', self.import_branch, no_commit=True)
self.log.info(
"""\
Replacing tree contents with those from the import branch:
git read-tree -u --reset %s
""", self.import_branch)
self.git.read_tree(self.import_branch, u=True, reset=True)
self.log.info(
"""\
Committing merge commit:
git commit --no-edit
""")
self.git.commit(no_edit=True)
# finally test that everything worked correctly by comparing if
# the tree object id's match
if self.git.rev_parse("HEAD^{tree}") != \
self.git.rev_parse("%s^{tree}" % self.import_branch):
raise ImportUpstreamError(
"Resulting tree does not match import")
except (GitCommandError, ImportUpstreamError):
self.log.error(
"""\
Failed to finish import by merging branch:
'%s'
into and replacing the contents of:
'%s'
""", self.import_branch, self.branch)
self._set_branch(self.branch, current_sha, force=True)
return False
except Exception:
self.log.exception("Unknown exception during finish")
self._set_branch(self.branch, current_sha, force=True)
raise
return True
class ImportStrategiesFactory(object):
__strategies = None
@classmethod
def create_strategy(cls, type, *args, **kwargs):
if type in cls.list_strategies():
return cls.__strategies[type](*args, **kwargs)
else:
raise RuntimeError("No class implements the requested strategy: "
"{0}".format(type))
@classmethod
def list_strategies(cls):
cls.__strategies = {
subclass._strategy: subclass
for subclass in LocateChangesStrategy.__subclasses__()
if subclass._strategy}
return cls.__strategies.keys()
class LocateChangesStrategy(GitMixin, Sequence):
"""Base locate changes strategy class
Needs to be extended with the specific strategy on how to handle changes
that are not yet upstream.
"""
__metaclass__ = ABCMeta
@abstractmethod
def __init__(self, git=None, *args, **kwargs):
"""Initialize an empty filters list"""
self.data = None
self.filters = []
super(LocateChangesStrategy, self).__init__(*args, **kwargs)
def __getitem__(self, key):
if not self.data:
self.data = self._popdata()
return self.data[key]
def __len__(self):
if not self.data:
self.data = self._popdata()
return len(self.data)
@classmethod
def get_strategy_name(cls):
return cls._strategy
def filtered_iter(self):
# chain the filters as generators so that we don't need to allocate new
# lists for each step in the filter chain.
commit_list = self
for f in self.filters:
commit_list = f.filter(commit_list)
return commit_list
def filtered_list(self):
return list(self.filtered_iter())
def _popdata(self):
"""Should return the list of commits from the searcher object"""
return self.searcher.list()
class LocateChangesWalk(LocateChangesStrategy):
_strategy = "drop"
def __init__(self, branch="HEAD", upstream="upstream/master",
search_refs=None, *args, **kwargs):
if not search_refs:
search_refs = []
search_refs.insert(0, upstream)
self.searcher = UpstreamMergeBaseSearcher(branch=branch,
patterns=search_refs)
self.upstream = upstream
super(LocateChangesWalk, self).__init__(*args, **kwargs)
def filtered_iter(self):
# may wish to make class used to remove duplicate objects configurable
# through git-upstream specific 'git config' settings
self.filters.append(
DiscardDuplicateGerritChangeId(self.upstream,
limit=self.searcher.commit))
self.filters.append(NoMergeCommitFilter())
self.filters.append(ReverseCommitFilter())
self.filters.append(DroppedCommitFilter())
self.filters.append(
SupersededCommitFilter(self.upstream,
limit=self.searcher.commit))
return super(LocateChangesWalk, self).filtered_iter()
class ImportCommand(LogDedentMixin, GitUpstreamCommand):
"""Import code from specified upstream branch.

149
git_upstream/commands/supersede.py
View File

@@ -15,157 +15,10 @@
# limitations under the License.
#
import re
from git import BadObject
from git import Head
from git_upstream.commands import GitUpstreamCommand
from git_upstream.errors import GitUpstreamError
from git_upstream.lib import note # noqa
from git_upstream.lib.searchers import CommitMessageSearcher
from git_upstream.lib.utils import GitMixin
from git_upstream.lib.supersede import Supersede
from git_upstream.log import LogDedentMixin
try:
from git import BadName
except ImportError:
BadName = None
class SupersedeError(GitUpstreamError):
"""Exception thrown by L{Supersede}"""
pass
class Supersede(LogDedentMixin, GitMixin):
"""
Mark a commit as superseded.
The mark is applied by means of a note in upstream-merge namespace
(refs/notes/upstream-merge).
The note will contain one or more of the following headers:
Superseded-by: I82ef79c3621dacf619a02404f16464877a06f158
"""
SUPERSEDE_HEADER = 'Superseded-by:'
NOTE_REF = 'refs/notes/upstream-merge'
CHANGE_ID_REGEX = '^I[0-9a-f]{6,40}$'
CHANGE_ID_HEADER_REGEX_FMT = '^Change-Id:\s*%s'
CHANGE_ID_HEADER_REGEX = '^Change-Id:\s*(I[0-9a-f]{6,40})$'
def __init__(self, git_object=None, change_ids=list(),
upstream_branch=None, force=False, *args, **kwargs):
# make sure to correctly initialize inherited objects before performing
# any computation
super(Supersede, self).__init__(*args, **kwargs)
# test commit parameter
if not git_object:
raise SupersedeError("Commit should be provided")
# test that we can use this git repo
if not self.is_detached():
raise SupersedeError("In 'detached HEAD' state")
# To Do: check if it possible and useful.
if self.repo.bare:
raise SupersedeError("Cannot add notes in bare repositories")
if not upstream_branch:
raise SupersedeError("Missing upstream_branch parameter")
try:
# test commit "id" presence
self._commit = self.repo.commit(git_object)
except (BadName, BadObject):
raise SupersedeError("Commit '%s' not found (or ambiguous)" %
git_object)
# test change_ids parameter
if len(change_ids) == 0:
raise SupersedeError("At least one change id should be provided")
self._upstream_branch = upstream_branch
self._change_ids = change_ids
git_branch = Head(self.repo, 'refs/heads/%s' % upstream_branch)
for change_id in change_ids:
# Check change id format
if not re.match(Supersede.CHANGE_ID_REGEX, change_id,
re.IGNORECASE):
raise SupersedeError("Invalid Change Id '%s'" % change_id)
# Check if change id is actually present in some commit
# reachable from <upstream_branch>
try:
change_commit = CommitMessageSearcher(
repo=self.repo, branch=git_branch,
pattern=Supersede.CHANGE_ID_HEADER_REGEX_FMT %
change_id).find()
self.log.debug("Change-id '%s' found in commit '%s'" %
(change_id, change_commit))
except RuntimeError:
if force:
self.log.warn("Warning: change-id '%s' not found in '%s'" %
(change_id, upstream_branch))
else:
raise SupersedeError(
"Change-Id '%s' not found in branch '%s'" %
(change_id, upstream_branch))
@property
def commit(self):
"""Commit to be marked as superseded."""
return self._commit
@property
def change_ids(self):
"""Change ids that make a commit obsolete."""
return self._change_ids
@property
def change_ids_branch(self):
"""Branch to search for change ids"""
return self._upstream_branch
def check_duplicates(self):
"""
Check if a supersede header is already present in the note containing
one of change ids passed on the command line
"""
new_note = self.commit.note(note_ref=Supersede.NOTE_REF)
if new_note:
pattern = '^%s\s?(%s)$' % (Supersede.SUPERSEDE_HEADER,
'|'.join(self.change_ids))
m = re.search(pattern, new_note, re.MULTILINE | re.IGNORECASE)
if m:
self.log.warning(
("Change-Id '%s' already present in the note for commit" +
" '%s'") % (m.group(1), self.commit))
return False
return True
def mark(self):
"""Create the note for the given commit with the given change-ids."""
self.log.debug("Creating a note for commit '%s'", self.commit)
if self.check_duplicates():
git_note = ''
for change_id in self.change_ids:
git_note += '%s %s\n' % (Supersede.SUPERSEDE_HEADER, change_id)
self.log.debug('With the following content:')
self.log.debug(git_note)
self.commit.append_note(git_note, note_ref=Supersede.NOTE_REF)
else:
self.log.warning('Note has not been added')
class SupersedeCommand(LogDedentMixin, GitUpstreamCommand):
"""Mark a commit as superseded by a set of change-ids.

122
git_upstream/lib/drop.py Normal file
View File

@@ -0,0 +1,122 @@
#
# Copyright (c) 2012-2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import re
from git import BadObject
from git_upstream.errors import GitUpstreamError
from git_upstream.lib.utils import GitMixin
from git_upstream.log import LogDedentMixin
try:
from git import BadName
except ImportError:
BadName = None
class DropError(GitUpstreamError):
"""Exception thrown by L{Drop}"""
pass
class Drop(LogDedentMixin, GitMixin):
"""Mark a commit to be dropped on next import.
Mark a commit as to be dropped.
The mark is applied by means of a note in upstream-merge namespace
(refs/notes/upstream-merge).
The note will contain the following header:
Dropped: Walter White <heisenberg@hp.com>
"""
DROP_HEADER = 'Dropped:'
NOTE_REF = 'refs/notes/upstream-merge'
def __init__(self, git_object=None, author=None, *args, **kwargs):
# make sure to correctly initialize inherited objects before performing
# any computation
super(Drop, self).__init__(*args, **kwargs)
# test parameters
if not git_object:
raise DropError("Commit should be provided")
try:
# test commit "id" presence
self._commit = self.repo.commit(git_object)
except (BadName, BadObject):
raise DropError(
"Commit '%s' not found (or ambiguous)" % git_object)
if not author:
self._author = '%s <%s>' % (self.repo.git.config('user.name'),
self.repo.git.config('user.email'))
else:
self._author = author
# test that we can use this git repo
if not self.is_detached():
raise DropError("In 'detached HEAD' state")
# To Do: check if it possible and useful.
if self.repo.bare:
raise DropError("Cannot add notes in bare repositories")
@property
def commit(self):
"""Commit to be marked as dropped."""
return self._commit
@property
def author(self):
"""Commit to be marked as dropped."""
return self._author
def check_duplicates(self):
"""Check if a dropped header is already present"""
note = self.commit.note(note_ref=Drop.NOTE_REF)
if note:
pattern = '^%s\s*(.+)' % Drop.DROP_HEADER
m = re.search(pattern, note, re.MULTILINE | re.IGNORECASE)
if m:
self.log.warning(
"""Drop header already present in the note for commit '%s':
%s""" % (self.commit, m.group(1)))
return False
return True
def mark(self):
"""
Create the note for the given commit with the given change-ids.
"""
self.log.debug("Creating a note for commit '%s'", self.commit)
if self.check_duplicates():
git_note = '%s %s\n' % (Drop.DROP_HEADER, self.author)
self.log.debug('With the following content:')
self.log.debug(git_note)
self.commit.append_note(git_note, note_ref=Drop.NOTE_REF)
else:
self.log.warning(
"Drop note has not been added as '%s' already has one" %
self.commit)

406
git_upstream/lib/importupstream.py Normal file
View File

@@ -0,0 +1,406 @@
#
# Copyright (c) 2012-2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from git import GitCommandError
from git_upstream.errors import GitUpstreamError
from git_upstream.lib.rebaseeditor import RebaseEditor
from git_upstream.lib.utils import GitMixin
from git_upstream.log import LogDedentMixin
class ImportUpstreamError(GitUpstreamError):
"""Exception thrown by L{ImportUpstream}"""
pass
class ImportUpstream(LogDedentMixin, GitMixin):
"""Import code from an upstream project and merge in additional branches
to create a new branch unto which changes that are not upstream but are
on the local branch are applied.
"""
def __init__(self, branch=None, upstream=None, import_branch=None,
extra_branches=None, *args, **kwargs):
if not extra_branches:
extra_branches = []
self._branch = branch
self._upstream = upstream
self._import_branch = import_branch
self._extra_branches = extra_branches
# make sure to correctly initialise inherited objects before performing
# any computation
super(ImportUpstream, self).__init__(*args, **kwargs)
# test that we can use this git repo
if not self.is_detached():
raise ImportUpstreamError("In 'detached HEAD' state")
if self.repo.bare:
raise ImportUpstreamError("Cannot perform imports in bare repos")
if self.branch == 'HEAD':
self._branch = str(self.repo.active_branch)
# validate branches exist and log all failures
branches = [
self.branch,
self.upstream
]
branches.extend(self.extra_branches)
invalid_ref = False
for branch in branches:
if not any(head for head in self.repo.heads
if head.name == branch):
msg = "Specified ref does not exist: '%s'"
self.log.error(msg, branch)
invalid_ref = True
if invalid_ref:
raise ImportUpstreamError("Invalid ref")
@property
def branch(self):
"""Branch to search for branch changes to apply when importing."""
return self._branch
@property
def upstream(self):
"""Branch containing the upstream project code base to track."""
return self._upstream
@property
def import_branch(self):
"""Pattern to use to generate the name, or user specified branch name
to use for import.
"""
return self._import_branch
@property
def extra_branches(self):
"""Branch containing the additional branches to be merged with the
upstream when importing.
"""
return self._extra_branches
def _set_branch(self, branch, commit, checkout=False, force=False):
if str(self.repo.active_branch) == branch:
self.log.info(
"""\
Resetting branch '%s' to specified commit '%s'
git reset --hard %s
""", branch, commit, commit)
self.git.reset(commit, hard=True)
elif checkout:
if force:
checkout_opt = '-B'
else:
checkout_opt = '-b'
self.log.info(
"""\
Checking out branch '%s' using specified commit '%s'
git checkout %s %s %s
""", branch, commit, checkout_opt, branch, commit)
self.git.checkout(checkout_opt, branch, commit)
else:
self.log.info(
"""\
Creating branch '%s' from specified commit '%s'
git branch --force %s %s
""", branch, commit, branch, commit)
self.git.branch(branch, commit, force=force)
def create_import(self, commit=None, import_branch=None, checkout=False,
force=False):
"""Create the import branch from the specified commit.
If the branch already exists abort if force is false
If current branch, reset the head to the specified commit
If checkout is true, switch and reset the branch to the commit
Otherwise just reset the branch to the specified commit
If the branch doesn't exist, create it and switch to it
automatically if checkout is true.
"""
if not commit:
commit = self.upstream
try:
self.git.show_ref(commit, quiet=True, heads=True)
except GitCommandError as e:
msg = "Invalid commit '%s' specified to import from"
self.log.error(msg, commit)
raise ImportUpstreamError((msg + ": %s"), commit, e)
if not import_branch:
import_branch = self.import_branch
# use describe in order to be certain about unique identifying 'commit'
# Create a describe string with the following format:
# <describe upstream>[-<extra branch abbref hash>]*
#
# Simply appends the 7 character ref abbreviation for each extra branch
# prefixed with '-', for each extra branch in the order they are given.
describe_commit = self.git.describe(commit, tags=True,
with_exceptions=False)
if not describe_commit:
self.log.warning("No tag describes the upstream branch")
describe_commit = self.git.describe(commit, always=True, tags=True)
self.log.info("""\
Using '%s' to describe:
%s
""", describe_commit, commit)
describe_branches = [describe_commit]
describe_branches.extend([self.git.rev_parse(b, short=True)
for b in self.extra_branches])
import_describe = "-".join(describe_branches)
self._import_branch = self.import_branch.format(
describe=import_describe)
self._import_branch = import_branch.format(describe=import_describe)
base = self._import_branch + "-base"
self.log.debug("Creating and switching to import branch base '%s' "
"created from '%s' (%s)", base, self.upstream, commit)
self.log.info(
"""\
Checking if import branch '%s' already exists:
git branch --list %s
""", base, base)
if self.git.show_ref("refs/heads/" + base, verify=True,
with_exceptions=False) and not force:
msg = "Import branch '%s' already exists, set 'force' to replace"
self.log.error(msg, self.import_branch)
raise ImportUpstreamError(msg % self.import_branch)
self._set_branch(base, commit, checkout, force)
if self.extra_branches:
self.log.info(
"""\
Merging additional branch(es) '%s' into import branch '%s'
git checkout %s
git merge %s
""", ", ".join(self.extra_branches), base, base,
" ".join(self.extra_branches))
self.git.checkout(base)
self.git.merge(*self.extra_branches)
def _linearise(self, branch, sequence, previous_import):
counter = len(sequence) - 1
ancestors = set()
self._set_branch(branch, previous_import, checkout=True, force=True)
root = previous_import.hexsha
while counter > 0:
# add commit to list of ancestors to check
ancestors.add(root)
# look for merge commits that are not part of ancestry path
for idx in xrange(counter - 1, -1, -1):
commit = sequence[idx]
# if there is only one parent, no need to check the others
if len(commit.parents) < 2:
ancestors.add(commit.hexsha)
elif any(p.hexsha not in ancestors for p in commit.parents):
self.log.debug("Rebase upto commit SHA1: %s",
commit.hexsha)
idx = idx + 1
break
else:
ancestors.add(commit.hexsha)
tip = sequence[idx].hexsha
self.log.info("Rebasing from %s to %s", root, tip)
previous = self.git.rev_parse(branch)
self.log.info("Rebasing onto '%s'", previous)
if root == previous and idx == 0:
# special case, we are already linear
self.log.info("Already in a linear layout")
return
self._set_branch(branch, tip, force=True)
try:
self.log.debug(
"""\
git rebase -p --onto=%s \\
%s %s
""", previous, root, branch)
self.git.rebase(root, branch, onto=previous, p=True)
except Exception:
self.git.rebase(abort=True, with_exceptions=False)
raise
counter = idx - 1
# set root commit for next loop
root = sequence[counter].hexsha
def apply(self, strategy, interactive=False):
"""Apply list of commits given onto latest import of upstream"""
commit_list = list(strategy.filtered_iter())
if len(commit_list) == 0:
self.log.notice("There are no local changes to be applied!")
return False
self.log.debug(
"""\
Should apply the following list of commits
%s
""", "\n ".join([c.hexsha for c in commit_list]))
base = self.import_branch + "-base"
self._set_branch(self.import_branch, self.branch, force=True)
self.log.info(
"""\
Creating import branch '%s' from specified commit '%s' in prep to
linearize the local changes before transposing to the new upstream:
git branch --force %s %s
""", self.import_branch, self.branch, self.import_branch,
self.branch)
self.log.notice("Attempting to linearise previous changes")
# attempt to silently linearize the current carried changes as a branch
# based on the previous located import commit. This provides a sane
# abort result for if the user needs to abort the rebase of this branch
# onto the new point upstream that was requested to import from.
try:
self._linearise(self.import_branch, strategy,
strategy.searcher.commit)
except Exception:
# Could ask user if they want to try and use the non clean route
# provided they don't mind that 'git rebase --abort' will result
# in a virtually useless local import branch
self.log.warning(
"""\
Exception occurred during linearisation of local changes on to
previous import to simplify behaviour should user need to abort
the rebase that applies these changes to the latest import
point. Attempting to tidy up state.
Do not Ctrl+C unless you wish to need to clean up your git
repository by hand.
""")
# reset head back to the tip of the changes to be rebased
self._set_branch(self.import_branch, self.branch, force=True)
rebase = RebaseEditor(interactive, repo=self.repo)
if len(commit_list):
first = commit_list[0]
self.log.info(
"""\
Rebase changes, dropping merges through editor:
git rebase --onto %s \\
%s %s
""", base, first.parents[0].hexsha, self.import_branch)
status, out, err = rebase.run(commit_list,
first.parents[0].hexsha,
self.import_branch,
onto=base)
if status:
if err and err.startswith("Nothing to do"):
# cancelled by user
self.log.notice("Cancelled by user")
return False
self.log.error("Rebase failed, will need user intervention to "
"resolve.")
if out:
self.log.notice(out)
if err:
self.log.notice(err)
# once we support resuming/finishing add a message here to tell
# the user to rerun this tool with the appropriate options to
# complete
return False
self.log.notice("Successfully applied all locally carried changes")
else:
self.log.warning("Warning, nothing to do: locally carried " +
"changes already rebased onto " + self.upstream)
return True
def resume(self, args):
"""Resume previous partial import"""
raise NotImplementedError
def finish(self):
"""Finish import
Finish the import by merging the import branch to the target while
performing suitable verification checks.
"""
self.log.info("No verification checks enabled")
self.git.checkout(self.branch)
current_sha = self.git.rev_parse("HEAD")
try:
self.log.info(
"""\
Merging by inverting the 'ours' strategy discard all changes
and replace existing branch contents with the new import.
""")
self.log.info(
"""\
Merging import branch to HEAD and ignoring changes:
git merge -s ours --no-commit %s
""", self.import_branch)
self.git.merge('-s', 'ours', self.import_branch, no_commit=True)
self.log.info(
"""\
Replacing tree contents with those from the import branch:
git read-tree -u --reset %s
""", self.import_branch)
self.git.read_tree(self.import_branch, u=True, reset=True)
self.log.info(
"""\
Committing merge commit:
git commit --no-edit
""")
self.git.commit(no_edit=True)
# finally test that everything worked correctly by comparing if
# the tree object id's match
if self.git.rev_parse("HEAD^{tree}") != \
self.git.rev_parse("%s^{tree}" % self.import_branch):
raise ImportUpstreamError(
"Resulting tree does not match import")
except (GitCommandError, ImportUpstreamError):
self.log.error(
"""\
Failed to finish import by merging branch:
'%s'
into and replacing the contents of:
'%s'
""", self.import_branch, self.branch)
self._set_branch(self.branch, current_sha, force=True)
return False
except Exception:
self.log.exception("Unknown exception during finish")
self._set_branch(self.branch, current_sha, force=True)
raise
return True

128
git_upstream/lib/strategies.py Normal file
View File

@@ -0,0 +1,128 @@
#
# Copyright (c) 2012-2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from abc import ABCMeta
from abc import abstractmethod
from collections import Sequence
from git_upstream.lib.searchers import DiscardDuplicateGerritChangeId
from git_upstream.lib.searchers import DroppedCommitFilter
from git_upstream.lib.searchers import NoMergeCommitFilter
from git_upstream.lib.searchers import ReverseCommitFilter
from git_upstream.lib.searchers import SupersededCommitFilter
from git_upstream.lib.searchers import UpstreamMergeBaseSearcher
from git_upstream.lib.utils import GitMixin
class ImportStrategiesFactory(object):
__strategies = None
@classmethod
def create_strategy(cls, type, *args, **kwargs):
if type in cls.list_strategies():
return cls.__strategies[type](*args, **kwargs)
else:
raise RuntimeError("No class implements the requested strategy: "
"{0}".format(type))
@classmethod
def list_strategies(cls):
cls.__strategies = {
subclass._strategy: subclass
for subclass in LocateChangesStrategy.__subclasses__()
if subclass._strategy}
return cls.__strategies.keys()
class LocateChangesStrategy(GitMixin, Sequence):
"""Base locate changes strategy class
Needs to be extended with the specific strategy on how to handle changes
that are not yet upstream.
"""
__metaclass__ = ABCMeta
@abstractmethod
def __init__(self, git=None, *args, **kwargs):
"""Initialize an empty filters list"""
self.data = None
self.filters = []
super(LocateChangesStrategy, self).__init__(*args, **kwargs)
def __getitem__(self, key):
if not self.data:
self.data = self._popdata()
return self.data[key]
def __len__(self):
if not self.data:
self.data = self._popdata()
return len(self.data)
@classmethod
def get_strategy_name(cls):
return cls._strategy
def filtered_iter(self):
# chain the filters as generators so that we don't need to allocate new
# lists for each step in the filter chain.
commit_list = self
for f in self.filters:
commit_list = f.filter(commit_list)
return commit_list
def filtered_list(self):
return list(self.filtered_iter())
def _popdata(self):
"""Should return the list of commits from the searcher object"""
return self.searcher.list()
class LocateChangesWalk(LocateChangesStrategy):
_strategy = "drop"
def __init__(self, branch="HEAD", upstream="upstream/master",
search_refs=None, *args, **kwargs):
if not search_refs:
search_refs = []
search_refs.insert(0, upstream)
self.searcher = UpstreamMergeBaseSearcher(branch=branch,
patterns=search_refs)
self.upstream = upstream
super(LocateChangesWalk, self).__init__(*args, **kwargs)
def filtered_iter(self):
# may wish to make class used to remove duplicate objects configurable
# through git-upstream specific 'git config' settings
self.filters.append(
DiscardDuplicateGerritChangeId(self.upstream,
limit=self.searcher.commit))
self.filters.append(NoMergeCommitFilter())
self.filters.append(ReverseCommitFilter())
self.filters.append(DroppedCommitFilter())
self.filters.append(
SupersededCommitFilter(self.upstream,
limit=self.searcher.commit))
return super(LocateChangesWalk, self).filtered_iter()

167
git_upstream/lib/supersede.py Normal file
View File

@@ -0,0 +1,167 @@
#
# Copyright (c) 2012-2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import re
from git import BadObject
from git import Head
from git_upstream.errors import GitUpstreamError
from git_upstream.lib import note # noqa
from git_upstream.lib.searchers import CommitMessageSearcher
from git_upstream.lib.utils import GitMixin
from git_upstream.log import LogDedentMixin
try:
from git import BadName
except ImportError:
BadName = None
class SupersedeError(GitUpstreamError):
"""Exception thrown by L{Supersede}"""
pass
class Supersede(LogDedentMixin, GitMixin):
"""
Mark a commit as superseded.
The mark is applied by means of a note in upstream-merge namespace
(refs/notes/upstream-merge).
The note will contain one or more of the following headers:
Superseded-by: I82ef79c3621dacf619a02404f16464877a06f158
"""
SUPERSEDE_HEADER = 'Superseded-by:'
NOTE_REF = 'refs/notes/upstream-merge'
CHANGE_ID_REGEX = '^I[0-9a-f]{6,40}$'
CHANGE_ID_HEADER_REGEX_FMT = '^Change-Id:\s*%s'
CHANGE_ID_HEADER_REGEX = '^Change-Id:\s*(I[0-9a-f]{6,40})$'
def __init__(self, git_object=None, change_ids=list(),
upstream_branch=None, force=False, *args, **kwargs):
# make sure to correctly initialize inherited objects before performing
# any computation
super(Supersede, self).__init__(*args, **kwargs)
# test commit parameter
if not git_object:
raise SupersedeError("Commit should be provided")
# test that we can use this git repo
if not self.is_detached():
raise SupersedeError("In 'detached HEAD' state")
# To Do: check if it possible and useful.
if self.repo.bare:
raise SupersedeError("Cannot add notes in bare repositories")
if not upstream_branch:
raise SupersedeError("Missing upstream_branch parameter")
try:
# test commit "id" presence
self._commit = self.repo.commit(git_object)
except (BadName, BadObject):
raise SupersedeError("Commit '%s' not found (or ambiguous)" %
git_object)
# test change_ids parameter
if len(change_ids) == 0:
raise SupersedeError("At least one change id should be provided")
self._upstream_branch = upstream_branch
self._change_ids = change_ids
git_branch = Head(self.repo, 'refs/heads/%s' % upstream_branch)
for change_id in change_ids:
# Check change id format
if not re.match(Supersede.CHANGE_ID_REGEX, change_id,
re.IGNORECASE):
raise SupersedeError("Invalid Change Id '%s'" % change_id)
# Check if change id is actually present in some commit
# reachable from <upstream_branch>
try:
change_commit = CommitMessageSearcher(
repo=self.repo, branch=git_branch,
pattern=Supersede.CHANGE_ID_HEADER_REGEX_FMT %
change_id).find()
self.log.debug("Change-id '%s' found in commit '%s'" %
(change_id, change_commit))
except RuntimeError:
if force:
self.log.warn("Warning: change-id '%s' not found in '%s'" %
(change_id, upstream_branch))
else:
raise SupersedeError(
"Change-Id '%s' not found in branch '%s'" %
(change_id, upstream_branch))
@property
def commit(self):
"""Commit to be marked as superseded."""
return self._commit
@property
def change_ids(self):
"""Change ids that make a commit obsolete."""
return self._change_ids
@property
def change_ids_branch(self):
"""Branch to search for change ids"""
return self._upstream_branch
def check_duplicates(self):
"""
Check if a supersede header is already present in the note containing
one of change ids passed on the command line
"""
new_note = self.commit.note(note_ref=Supersede.NOTE_REF)
if new_note:
pattern = '^%s\s?(%s)$' % (Supersede.SUPERSEDE_HEADER,
'|'.join(self.change_ids))
m = re.search(pattern, new_note, re.MULTILINE | re.IGNORECASE)
if m:
self.log.warning(
("Change-Id '%s' already present in the note for commit" +
" '%s'") % (m.group(1), self.commit))
return False
return True
def mark(self):
"""Create the note for the given commit with the given change-ids."""
self.log.debug("Creating a note for commit '%s'", self.commit)
if self.check_duplicates():
git_note = ''
for change_id in self.change_ids:
git_note += '%s %s\n' % (Supersede.SUPERSEDE_HEADER, change_id)
self.log.debug('With the following content:')
self.log.debug(git_note)
self.commit.append_note(git_note, note_ref=Supersede.NOTE_REF)
else:
self.log.warning('Note has not been added')

11
git_upstream/tests/test_drop.py
View File

@@ -17,7 +17,8 @@
from git import GitCommandError
from git_upstream.commands import drop as d
from git_upstream.lib.drop import Drop
from git_upstream.lib.drop import DropError
from git_upstream.tests import base
@@ -37,22 +38,22 @@ class TestDrop(base.BaseTestCase):
automatic_author = '%s <%s>' % (self.repo.git.config('user.name'),
self.repo.git.config('user.email'))
t = d.Drop(git_object=self.first_commit)
t = Drop(git_object=self.first_commit)
self.assertEqual(t.author, automatic_author)
t = d.Drop(git_object=self.first_commit, author=self.author)
t = Drop(git_object=self.first_commit, author=self.author)
self.assertEqual(t.author, self.author)
def test_invalid_commit(self):
"""Test drop initialization with invalid commit"""
self.assertRaises(d.DropError, d.Drop,
self.assertRaises(DropError, Drop,
git_object=self.invalid_commit)
def test_mark(self):
"""Test drop mark"""
t = d.Drop(git_object=self.first_commit, author=self.author)
t = Drop(git_object=self.first_commit, author=self.author)
try:
# Older git versions don't support --ignore-missing so we need to

29
git_upstream/tests/test_supersede.py
View File

@@ -17,7 +17,8 @@
from git import GitCommandError
from git_upstream.commands import supersede as s
from git_upstream.lib.supersede import Supersede
from git_upstream.lib.supersede import SupersedeError
from git_upstream.tests import base
@@ -46,9 +47,9 @@ class TestSupersede(base.BaseTestCase):
def test_valid_parameters(self):
"""Test supersede initialization and read properties"""
t = s.Supersede(git_object=self.first_commit,
change_ids=self.first_change_ids,
upstream_branch=self.change_ids_branch)
t = Supersede(git_object=self.first_commit,
change_ids=self.first_change_ids,
upstream_branch=self.change_ids_branch)
self.assertEqual(t.commit, self.first_commit)
self.assertNotEqual(t.commit, self.second_commit)
@@ -60,7 +61,7 @@ class TestSupersede(base.BaseTestCase):
def test_invalid_commit(self):
"""Test supersede initialization with invalid commit"""
self.assertRaises(s.SupersedeError, s.Supersede,
self.assertRaises(SupersedeError, Supersede,
git_object=self.invalid_commit,
change_ids=self.first_change_ids,
upstream_branch=self.change_ids_branch)
@@ -68,9 +69,9 @@ class TestSupersede(base.BaseTestCase):
def test_multiple_change_id(self):
"""Test supersede initialization with multiple change ids"""
t = s.Supersede(git_object=self.first_commit,
change_ids=self.second_change_ids,
upstream_branch=self.change_ids_branch)
t = Supersede(git_object=self.first_commit,
change_ids=self.second_change_ids,
upstream_branch=self.change_ids_branch)
self.assertEqual(t.commit, self.first_commit)
self.assertNotEqual(t.commit, self.second_commit)
@@ -78,7 +79,7 @@ class TestSupersede(base.BaseTestCase):
def test_invalid_cids(self):
"""Test supersede initialization with invalid cids"""
self.assertRaises(s.SupersedeError, s.Supersede,
self.assertRaises(SupersedeError, Supersede,
git_object=self.first_commit,
change_ids=self.invalid_change_ids,
upstream_branch=self.change_ids_branch)
@@ -86,7 +87,7 @@ class TestSupersede(base.BaseTestCase):
def test_default_upstream_branch(self):
"""Test supersede initialization with no branch name"""
self.assertRaises(s.SupersedeError, s.Supersede,
self.assertRaises(SupersedeError, Supersede,
git_object=self.first_commit,
change_ids=self.invalid_change_ids,
upstream_branch=self.invalid_change_ids_branch)
@@ -94,16 +95,16 @@ class TestSupersede(base.BaseTestCase):
def test_no_upstream_branch(self):
"""Test supersede initialization with invalid branch name"""
self.assertRaises(s.SupersedeError, s.Supersede,
self.assertRaises(SupersedeError, Supersede,
git_object=self.first_commit,
change_ids=self.invalid_change_ids)
def test_mark(self):
"""Test Supersede mark"""
t = s.Supersede(git_object=self.first_commit,
change_ids=self.first_change_ids,
upstream_branch=self.change_ids_branch)
t = Supersede(git_object=self.first_commit,
change_ids=self.first_change_ids,
upstream_branch=self.change_ids_branch)
try:
# Older git versions don't support --ignore-missing