From fa9bd28914a2c2e997b7a69f11974337f379cce1 Mon Sep 17 00:00:00 2001 From: Darragh Bailey Date: Wed, 22 May 2019 15:31:53 +0100 Subject: [PATCH] Split code for reuse and refactor for code style To make it easier to reuse GitTree elsewhere move outside of the gitfixture and reference. Instead of treating the functions in utils as private to indicate they shouldn't be relied upon outside of the package, switch utils to indicate that it should not relied upon to clear up some IDE warnings on use of private functions. Finally avoid use of potentially mutable default params. Change-Id: I3365a2c49ff570bd9e5eba5f53b4b3908dddd2b6 --- fixtures_git/{utils.py => _utils.py} | 6 +- fixtures_git/gitfixture.py | 166 ++----------------------- fixtures_git/gittree.py | 176 +++++++++++++++++++++++++++ tests/unit/utils/test_toposort.py | 12 +- 4 files changed, 192 insertions(+), 168 deletions(-) rename fixtures_git/{utils.py => _utils.py} (96%) create mode 100644 fixtures_git/gittree.py diff --git a/fixtures_git/utils.py b/fixtures_git/_utils.py similarity index 96% rename from fixtures_git/utils.py rename to fixtures_git/_utils.py index aef5628..ba69b61 100644 --- a/fixtures_git/utils.py +++ b/fixtures_git/_utils.py @@ -16,7 +16,7 @@ import re -def _get_node_to_pick(node): +def get_node_to_pick(node): m = re.search(r'(.*)(\d+)$', node) if m: # get copy of a another change @@ -33,7 +33,7 @@ _VISITED = 1 _FINISHED = 2 -def _reverse_toposort(data): +def reverse_toposort(data): # convert to dict for linear lookup times when returning data = dict(data) @@ -63,7 +63,7 @@ def _reverse_toposort(data): visited[node] = _VISITED nodes_to_visit.append(node) # special case for cherry-picking changes - c_node = _get_node_to_pick(node) + c_node = get_node_to_pick(node) if c_node and c_node not in visited: nodes_to_visit.append(c_node) diff --git a/fixtures_git/gitfixture.py b/fixtures_git/gitfixture.py index 60bac0a..879cda3 100644 --- a/fixtures_git/gitfixture.py +++ b/fixtures_git/gitfixture.py @@ -13,15 +13,15 @@ # License for the specific language governing permissions and limitations # under the License. -import faker import os import shutil import tempfile +import faker import fixtures import git -from fixtures_git import utils +from . import gittree try: import urlparse as parse @@ -29,161 +29,6 @@ except ImportError: from urllib import parse -class GitTree(object): - """Helper class to build a git repository from a graph definition - - Helper class to build a git repository from a graph definition of - nodes and their parent nodes. A list of branches may be provided - where each element has two members corresponding to the name and the - target node it references. - - Supports unordered graphs, only requirement is that there is a commit - defined with no parents, which will become the root commit. - - Root commits can specified by an empty list as the second member: - - ('NodeA', []) - - Merge commits are specified by multiple nodes: - - ('NodeMerge', ['Node1', 'Node2']) - - - An '=' prefix can be used in front of one of the parents of a merge - commit to indicate that the merge commit's tree should be copied - directly from that parent ignoring any contributions from the other - parents' trees (like what git enables with the '-s ours' merge - strategy. - - This is used when needing to preserve history but not interested in - the changes from the other branches. One tool that makes extensive - usage of this is _git-upstream. - - E.g., the following will result in a merge commit 'C', with parents - 'P1' and 'P2', but will have the same tree as 'P1'. - - ('C', ['=P1', 'P2']) - - - The tree building code can handle a graph definition being out of - order but will fail to find certain circular dependencies and may - result in an infinite loop. - - Examples: - - [('A', []), ('B', ['A']), ('C', ['B'])] - [('A', []), ('C', ['B']), ('B', ['A'])] - - .. _git-upstream: https://pypi.python.org/pypi/git-upstream - """ - - def __init__(self, gitrepo, tree, branches): - self.graph = {} - self.gitrepo = gitrepo - self.repo = gitrepo.repo - self._build_git_tree(tree, branches) - - def _commit(self, node): - p_node = utils._get_node_to_pick(node) - if p_node: - self.repo.git.cherry_pick(self.graph[p_node], x=True) - else: - # standard commit - self.gitrepo.add_commits(1, ref="HEAD", - message_prefix="[%s]" % node) - - def _merge_commit(self, node, parents): - # merge commits - parent_nodes = [p.lstrip("=") for p in parents] - commits = [str(self.graph[p]) for p in parent_nodes[1:]] - - if any([p.startswith("=") for p in parents]): - # special merge commit using inverse of 'ours' by - # emptying the current index and then reading in any - # trees of the nodes prefixed with '=' - use = [str(self.graph[p.lstrip("=")]) - for p in parents if p.startswith("=")] - try: - self.repo.git.merge(*commits, s="ours", no_commit=True) - except git.exc.GitCommandError as exc: - if 'refusing to merge unrelated histories' in exc.stderr: - self.repo.git.merge(*commits, s="ours", no_commit=True, - allow_unrelated_histories=True) - else: - raise - self.repo.git.read_tree(empty=True) - self.repo.git.read_tree(empty=True) - self.repo.git.read_tree(*use, u=True, reset=True) - elif len(commits) < 2: - # standard merge - try: - self.repo.git.merge(*commits, no_commit=True) - except git.exc.GitCommandError as exc: - if 'refusing to merge unrelated histories' in exc.stderr: - self.repo.git.merge(*commits, no_commit=True, - allow_unrelated_histories=True) - else: - raise - else: - # multi-branch merge, git is not great at handling - # merging multiple orphaned branches - try: - self.repo.git.merge(*commits, s="ours", no_commit=True) - except git.exc.GitCommandError as exc: - if 'refusing to merge unrelated histories' in exc.stderr: - self.repo.git.merge(*commits, s="ours", no_commit=True, - allow_unrelated_histories=True) - else: - raise - self.repo.git.read_tree(empty=True) - self.repo.git.read_tree("HEAD", *commits) - self.repo.git.checkout("--", ".") - self.repo.git.commit(m="[%s] Merging %s into %s" % - (node, ",".join(parent_nodes[1:]), - parent_nodes[0])) - self.repo.git.clean(f=True, d=True, x=True) - - def _build_git_tree(self, graph_def, branches=[]): - - # require that graphs must have at least 1 node with no - # parents, which is a root commit in git - if not any([True for _, parents in graph_def if not parents]): - assert("No root commit defined in test graph") - - for node, parents in utils._reverse_toposort(graph_def): - if not parents: - # root commit - self.repo.git.symbolic_ref("HEAD", "refs/heads/%s" % node) - self.repo.git.rm(".", r=True, cached=True, - with_exceptions=False) - self.repo.git.clean(f=True, d=True, x=True) - self.gitrepo.add_commits(1, ref="HEAD", - message_prefix="[%s]" % node) - # only explicitly listed branches should exist afterwards - self.repo.git.checkout(self.repo.commit()) - self.repo.git.branch(node, D=True) - - else: - # checkout the dependent node - self.repo.git.checkout(self.graph[parents[0].lstrip('=')]) - if len(parents) > 1: - # merge commits - self._merge_commit(node, parents) - else: - self._commit(node) - self.graph[node] = self.repo.commit() - - for name, node in branches: - self.repo.git.branch(name, str(self.graph[node]), f=True) - - # return to master - self.repo.git.checkout("master") - - def commits_from_nodes(self, nodes=[]): - - return [self.graph[n] for n in nodes] - - class GitFixture(fixtures.Fixture): """Create a git repo in which to operate. @@ -243,7 +88,7 @@ class GitFixture(fixtures.Fixture): self.repo.git.commit(m="Initialize empty repo", allow_empty=True) if self._graph: - self.gittree = GitTree(self, self._graph, self._branches) + self.gittree = gittree.GitTree(self, self._graph, self._branches) def _create_file(self): contents = "\n\n".join(self._faker.paragraphs(3)) @@ -276,9 +121,12 @@ class GitFixture(fixtures.Fixture): message = message + "\n\nChange-Id: %s" % change_id self.repo.git.commit(m=message) - def add_commits(self, num=None, ref="HEAD", change_ids=[], + def add_commits(self, num=None, ref="HEAD", change_ids=None, message_prefix=None): """Create the given number of commits using generated files""" + if change_ids is None: + change_ids = [] + if ref != "HEAD": self.repo.git.checkout(ref) diff --git a/fixtures_git/gittree.py b/fixtures_git/gittree.py new file mode 100644 index 0000000..221633c --- /dev/null +++ b/fixtures_git/gittree.py @@ -0,0 +1,176 @@ +# Copyright (c) 2018 Hewlett Packard Enterprise Development Company LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import git + +from fixtures_git import _utils + + +class GitTree(object): + """Helper class to build a git repository from a graph definition + + Helper class to build a git repository from a graph definition of + nodes and their parent nodes. A list of branches may be provided + where each element has two members corresponding to the name and the + target node it references. + + Supports unordered graphs, only requirement is that there is a commit + defined with no parents, which will become the root commit. + + Root commits can specified by an empty list as the second member: + + ('NodeA', []) + + Merge commits are specified by multiple nodes: + + ('NodeMerge', ['Node1', 'Node2']) + + + An '=' prefix can be used in front of one of the parents of a merge + commit to indicate that the merge commit's tree should be copied + directly from that parent ignoring any contributions from the other + parents' trees (like what git enables with the '-s ours' merge + strategy. + + This is used when needing to preserve history but not interested in + the changes from the other branches. One tool that makes extensive + usage of this is _git-upstream. + + E.g., the following will result in a merge commit 'C', with parents + 'P1' and 'P2', but will have the same tree as 'P1'. + + ('C', ['=P1', 'P2']) + + + The tree building code can handle a graph definition being out of + order but will fail to find certain circular dependencies and may + result in an infinite loop. + + Examples: + + [('A', []), ('B', ['A']), ('C', ['B'])] + [('A', []), ('C', ['B']), ('B', ['A'])] + + .. _git-upstream: https://pypi.python.org/pypi/git-upstream + """ + + def __init__(self, gitrepo, tree, branches): + self.graph = {} + self.gitrepo = gitrepo + self.repo = gitrepo.repo + self._build_git_tree(tree, branches) + + def _commit(self, node): + p_node = _utils.get_node_to_pick(node) + if p_node: + self.repo.git.cherry_pick(self.graph[p_node], x=True) + else: + # standard commit + self.gitrepo.add_commits(1, ref="HEAD", + message_prefix="[%s]" % node) + + def _merge_commit(self, node, parents): + # merge commits + parent_nodes = [p.lstrip("=") for p in parents] + commits = [str(self.graph[p]) for p in parent_nodes[1:]] + + if any([p.startswith("=") for p in parents]): + # special merge commit using inverse of 'ours' by + # emptying the current index and then reading in any + # trees of the nodes prefixed with '=' + use = [str(self.graph[p.lstrip("=")]) + for p in parents if p.startswith("=")] + try: + self.repo.git.merge(*commits, s="ours", no_commit=True) + except git.exc.GitCommandError as exc: + if 'refusing to merge unrelated histories' in exc.stderr: + self.repo.git.merge(*commits, s="ours", no_commit=True, + allow_unrelated_histories=True) + else: + raise + self.repo.git.read_tree(empty=True) + self.repo.git.read_tree(empty=True) + self.repo.git.read_tree(*use, u=True, reset=True) + elif len(commits) < 2: + # standard merge + try: + self.repo.git.merge(*commits, no_commit=True) + except git.exc.GitCommandError as exc: + if 'refusing to merge unrelated histories' in exc.stderr: + self.repo.git.merge(*commits, no_commit=True, + allow_unrelated_histories=True) + else: + raise + else: + # multi-branch merge, git is not great at handling + # merging multiple orphaned branches + try: + self.repo.git.merge(*commits, s="ours", no_commit=True) + except git.exc.GitCommandError as exc: + if 'refusing to merge unrelated histories' in exc.stderr: + self.repo.git.merge(*commits, s="ours", no_commit=True, + allow_unrelated_histories=True) + else: + raise + self.repo.git.read_tree(empty=True) + self.repo.git.read_tree("HEAD", *commits) + self.repo.git.checkout("--", ".") + self.repo.git.commit(m="[%s] Merging %s into %s" % + (node, ",".join(parent_nodes[1:]), + parent_nodes[0])) + self.repo.git.clean(f=True, d=True, x=True) + + def _build_git_tree(self, graph_def, branches=None): + + if branches is None: + branches = [] + + # require that graphs must have at least 1 node with no + # parents, which is a root commit in git + if not any([True for _, parents in graph_def if not parents]): + assert "No root commit defined in test graph" + + for node, parents in _utils.reverse_toposort(graph_def): + if not parents: + # root commit + self.repo.git.symbolic_ref("HEAD", "refs/heads/%s" % node) + self.repo.git.rm(".", r=True, cached=True, + with_exceptions=False) + self.repo.git.clean(f=True, d=True, x=True) + self.gitrepo.add_commits(1, ref="HEAD", + message_prefix="[%s]" % node) + # only explicitly listed branches should exist afterwards + self.repo.git.checkout(self.repo.commit()) + self.repo.git.branch(node, D=True) + + else: + # checkout the dependent node + self.repo.git.checkout(self.graph[parents[0].lstrip('=')]) + if len(parents) > 1: + # merge commits + self._merge_commit(node, parents) + else: + self._commit(node) + self.graph[node] = self.repo.commit() + + for name, node in branches: + self.repo.git.branch(name, str(self.graph[node]), f=True) + + # return to master + self.repo.git.checkout("master") + + def commits_from_nodes(self, nodes): + + return [self.graph[n] for n in nodes] diff --git a/tests/unit/utils/test_toposort.py b/tests/unit/utils/test_toposort.py index d559714..cdddef5 100644 --- a/tests/unit/utils/test_toposort.py +++ b/tests/unit/utils/test_toposort.py @@ -16,7 +16,7 @@ import testtools -from fixtures_git import utils +from fixtures_git import _utils from tests import base @@ -30,7 +30,7 @@ class TestResolve(testtools.TestCase): ] self.assertEqual( nodes, - list(utils._reverse_toposort(nodes)), + list(_utils.reverse_toposort(nodes)), ) def test_unordered(self): @@ -41,7 +41,7 @@ class TestResolve(testtools.TestCase): ] self.assertEqual( [('A', []), ('B', ['A']), ('C', ['B'])], - list(utils._reverse_toposort(nodes)) + list(_utils.reverse_toposort(nodes)) ) def test_merge(self): @@ -52,7 +52,7 @@ class TestResolve(testtools.TestCase): ('E', ['D', 'C']), ('D', ['A']), ] - sorted = list(utils._reverse_toposort(nodes)) + sorted = list(_utils.reverse_toposort(nodes)) self.assertThat( (nodes[2], nodes[0], nodes[1], nodes[3]), base.IsOrderedSubsetOf(sorted) @@ -72,7 +72,7 @@ class TestResolve(testtools.TestCase): ('G', ['F', 'C']), ('F', ['A']), ] - sorted = list(utils._reverse_toposort(nodes)) + sorted = list(_utils.reverse_toposort(nodes)) # A -> B -> C -> E self.assertThat( (nodes[2], nodes[0], nodes[1], nodes[3]), @@ -101,7 +101,7 @@ class TestResolve(testtools.TestCase): ('A', []), # root commit ('D', ['B', 'C']), ] - sorted = list(utils._reverse_toposort(nodes)) + sorted = list(_utils.reverse_toposort(nodes)) # assert partial ordering because nodes A & B may come # before or after node C. Just make sure that node D # is defined after them.