diff --git a/reno/loader.py b/reno/loader.py index 5d70cdb..f11196e 100644 --- a/reno/loader.py +++ b/reno/loader.py @@ -52,7 +52,7 @@ class Loader(object): self._earliest_version = conf.earliest_version self._cache = None - self._scanner = scanner.Scanner(self._config) + self._scanner = None self._scanner_output = None self._cache_filename = get_cache_filename(self._reporoot, self._notespath) @@ -76,6 +76,7 @@ class Loader(object): for n in self._cache['notes'] } else: + self._scanner = scanner.Scanner(self._config) self._scanner_output = self._scanner.get_notes_by_version() @property diff --git a/reno/scanner.py b/reno/scanner.py index cd35291..86f3791 100644 --- a/reno/scanner.py +++ b/reno/scanner.py @@ -20,6 +20,9 @@ import re import subprocess import sys +from dulwich import refs +from dulwich import repo + from reno import utils LOG = logging.getLogger(__name__) @@ -78,40 +81,48 @@ PRE_RELEASE_RE = re.compile(''' ''', flags=re.VERBOSE | re.UNICODE) -def _get_version_tags_on_branch(reporoot, branch): - """Return tags from the branch, in date order. - - Need to get the list of tags in the right order, because the topo - search breaks the date ordering. Use git to ask for the tags in - order, rather than trying to sort them, because many repositories - have "non-standard" tags or have renumbered projects (from - date-based to SemVer), for which sorting would require complex - logic. - - """ - tags = [] - tag_cmd = [ - 'git', 'log', - '--simplify-by-decoration', - '--pretty="%d"', - ] - if branch: - tag_cmd.append(branch) - LOG.debug('running %s' % ' '.join(tag_cmd)) - tag_results = utils.check_output(tag_cmd, cwd=reporoot) - LOG.debug(tag_results) - for line in tag_results.splitlines(): - LOG.debug('line %r' % line) - for match in TAG_RE.findall(line): - tags.append(match) - return tags - - class Scanner(object): def __init__(self, conf): self.conf = conf self.reporoot = self.conf.reporoot + self._repo = repo.Repo(self.reporoot) + self._load_tags() + + def _load_tags(self): + self._all_tags = { + k.partition(b'/tags/')[-1].decode('utf-8'): v + for k, v in self._repo.get_refs().items() + if k.startswith(b'refs/tags/') + } + self._shas_to_tags = {} + for tag, tag_sha in self._all_tags.items(): + # The tag has its own SHA, but the tag refers to the commit and + # that's the SHA we'll see when we scan commits on a branch. + tag_obj = self._repo[tag_sha] + tagged_sha = tag_obj.object[1] + self._shas_to_tags.setdefault(tagged_sha, []).append(tag) + + def _get_tags_on_branch(self, branch): + "Return a list of tag names on the given branch." + results = [] + if branch: + branch_ref = b'refs/heads/' + branch.encode('utf-8') + if not refs.check_ref_format(branch_ref): + raise ValueError( + '{!r} does not look like a valid branch reference'.format( + branch_ref)) + branch_head = self._repo.refs[branch_ref] + else: + branch_head = self._repo.refs[b'HEAD'] + w = self._repo.get_walker(branch_head) + for c in w: + # shas_to_tags has encoded versions of the shas + # but the commit object gives us a decoded version + sha = c.commit.sha().hexdigest().encode('ascii') + if sha in self._shas_to_tags: + results.extend(self._shas_to_tags[sha]) + return results def _get_current_version(self, branch=None): """Return the current version of the repository. @@ -252,7 +263,7 @@ class Scanner(object): # order. We scan the commit history in topological order to ensure # we have the commits in the right version, so we might encounter # the tags in a different order during that phase. - versions_by_date = _get_version_tags_on_branch(reporoot, branch) + versions_by_date = self._get_tags_on_branch(branch) LOG.debug('versions by date %r' % (versions_by_date,)) versions = [] earliest_seen = collections.OrderedDict() diff --git a/reno/tests/test_scanner.py b/reno/tests/test_scanner.py index 5869a20..3696364 100644 --- a/reno/tests/test_scanner.py +++ b/reno/tests/test_scanner.py @@ -19,11 +19,9 @@ import logging import os.path import re import subprocess -import textwrap import unittest import fixtures -import mock from testtools.content import text_content from reno import config @@ -197,7 +195,6 @@ class Base(base.TestCase): self.temp_dir = self.useFixture(fixtures.TempDir()).path self.reporoot = os.path.join(self.temp_dir, 'reporoot') self.c = config.Config(self.reporoot) - self.scanner = scanner.Scanner(self.c) self._git_setup() self._counter = itertools.count(1) self.get_note_num = lambda: next(self._counter) @@ -207,6 +204,7 @@ class BasicTest(Base): def test_non_python_no_tags(self): filename = self._add_notes_file() + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -220,6 +218,7 @@ class BasicTest(Base): def test_python_no_tags(self): self._make_python_package() filename = self._add_notes_file() + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -234,6 +233,7 @@ class BasicTest(Base): filename = self._add_notes_file() self._add_other_file('not-a-release-note.txt') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -247,6 +247,7 @@ class BasicTest(Base): def test_note_commit_tagged(self): filename = self._add_notes_file() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -261,6 +262,7 @@ class BasicTest(Base): self._make_python_package() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') filename = self._add_notes_file() + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -276,6 +278,7 @@ class BasicTest(Base): self._add_other_file('ignore-1.txt') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') self._add_other_file('ignore-2.txt') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -291,6 +294,7 @@ class BasicTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') f1 = self._add_notes_file() f2 = self._add_notes_file() + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -306,6 +310,7 @@ class BasicTest(Base): f1 = self._add_notes_file(commit=False) f2 = self._add_notes_file() self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -322,6 +327,7 @@ class BasicTest(Base): f1 = self._add_notes_file() self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') f2 = self._add_notes_file() + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -342,6 +348,7 @@ class BasicTest(Base): f2 = f1.replace('slug1', 'slug2') self._run_git('mv', f1, f2) self._git_commit('rename note file') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -361,6 +368,7 @@ class BasicTest(Base): f2 = f1.replace('slug1', 'slug0') self._run_git('mv', f1, f2) self._git_commit('rename note file') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -380,6 +388,7 @@ class BasicTest(Base): with open(os.path.join(self.reporoot, f1), 'w') as f: f.write('---\npreamble: new contents for file') self._git_commit('edit note file') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -399,6 +408,7 @@ class BasicTest(Base): f2 = f1.replace('slug1', 'slug2') self._run_git('mv', f1, f2) self._git_commit('rename note file') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -421,6 +431,7 @@ class BasicTest(Base): 'slug1-0000000000000001') self._run_git('mv', f1, f2) self._git_commit('rename note file') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -443,6 +454,7 @@ class BasicTest(Base): self.c.override( earliest_version='2.0.0', ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -463,6 +475,7 @@ class BasicTest(Base): self._run_git('rm', f1) self._git_commit('remove note file') self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -493,6 +506,7 @@ class BasicTest(Base): '--pretty=%H %d', '--name-only') self.addDetail('git log', text_content(log_results)) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -512,6 +526,7 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a1') f1 = self._add_notes_file('slug1') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a2') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -528,6 +543,7 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b1') f1 = self._add_notes_file('slug1') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b2') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -544,6 +560,7 @@ class PreReleaseTest(Base): self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc1') f1 = self._add_notes_file('slug1') self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc2') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -569,6 +586,7 @@ class PreReleaseTest(Base): self.c.override( collapse_pre_releases=True, ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -591,6 +609,7 @@ class PreReleaseTest(Base): self.c.override( collapse_pre_releases=True, ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -616,6 +635,7 @@ class PreReleaseTest(Base): self.c.override( collapse_pre_releases=True, ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -644,6 +664,7 @@ class MergeCommitTest(Base): self._run_git('merge', '--no-ff', 'test_merge_commit') self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -672,6 +693,7 @@ class MergeCommitTest(Base): self._run_git('merge', '--no-ff', 'test_merge_commit') self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'second tag', '2.0.0') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -704,6 +726,7 @@ class MergeCommitTest(Base): self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') self._add_other_file('ignore-3.txt') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -740,6 +763,7 @@ class MergeCommitTest(Base): self._add_other_file('ignore-2.txt') self._run_git('tag', '-s', '-m', 'third tag', '2.0.0') self._add_other_file('ignore-3.txt') + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -788,8 +812,9 @@ class BranchTest(Base): self._run_git('checkout', '2.0.0') self._run_git('checkout', '-b', 'stable/2') f21 = self._add_notes_file('slug21') - log_text = self._run_git('log') + log_text = self._run_git('log', '--decorate') self.addDetail('git log', text_content(log_text)) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -815,6 +840,7 @@ class BranchTest(Base): self.c.override( branch='stable/2', ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -842,6 +868,7 @@ class BranchTest(Base): self.c.override( stop_at_branch_base=False, ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -878,6 +905,7 @@ class BranchTest(Base): branch='stable/4', collapse_pre_releases=False, ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -914,6 +942,7 @@ class BranchTest(Base): branch='stable/4', collapse_pre_releases=True, ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -947,6 +976,7 @@ class BranchTest(Base): self.c.override( branch='stable/4', ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -980,6 +1010,7 @@ class BranchTest(Base): self.c.override( branch='stable/4', ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -1011,6 +1042,7 @@ class BranchTest(Base): self.c.override( branch='stable/4', ) + self.scanner = scanner.Scanner(self.c) raw_results = self.scanner.get_notes_by_version() results = { k: [f for (f, n) in v] @@ -1024,166 +1056,25 @@ class BranchTest(Base): ) -class GetTagsParseTest(base.TestCase): +class TagsTest(Base): - EXPECTED = [ - '2.0.0', - '1.8.1', - '1.8.0', - '1.7.1', - '1.7.0', - '1.6.0', - '1.5.0', - '1.4.0', - '1.3.0', - '1.2.0', - '1.1.0', - '1.0.0', - '0.11.2', - '0.11.1', - '0.11.0', - '0.10.1', - '0.10.0', - '0.9.0', - '0.8.0', - '0.7.1', - '0.7.0', - '0.6.0', - '0.5.1', - '0.5.0', - '0.4.2', - '0.4.1', - '0.4.0', - '0.3.2', - '0.3.1', - '0.3.0', - '0.2.5', - '0.2.4', - '0.2.3', - '0.2.2', - '0.2.1', - '0.2.0', - '0.1.3', - '0.1.2', - '0.1.1', - '0.1.0', - ] + def setUp(self): + super(TagsTest, self).setUp() + self._make_python_package() + self.f1 = self._add_notes_file('slug1') + self._run_git('tag', '-s', '-m', 'first tag', '1.0.0') + self.f2 = self._add_notes_file('slug2') + self._run_git('tag', '-s', '-m', 'first tag', '2.0.0') + self._add_notes_file('slug3') + self._run_git('tag', '-s', '-m', 'first tag', '3.0.0') - def test_keystoneclient_ubuntu_1_9_1(self): - # git 1.9.1 as it produces output on ubuntu for python-keystoneclient - # git log --simplify-by-decoration --pretty="%d" - tag_list_output = textwrap.dedent(""" - (HEAD, origin/master, origin/HEAD, gerrit/master, master) - (apu/master) - (tag: 2.0.0) - (tag: 1.8.1) - (tag: 1.8.0) - (tag: 1.7.1) - (tag: 1.7.0) - (tag: 1.6.0) - (tag: 1.5.0) - (tag: 1.4.0) - (uncap-requirements) - (tag: 1.3.0) - (tag: 1.2.0) - (tag: 1.1.0) - (tag: 1.0.0) - (tag: 0.11.2) - (tag: 0.11.1) - (tag: 0.11.0) - (tag: 0.10.1) - (tag: 0.10.0) - (tag: 0.9.0) - (tag: 0.8.0) - (tag: 0.7.1) - (tag: 0.7.0) - (tag: 0.6.0) - (tag: 0.5.1) - (tag: 0.5.0) - (tag: 0.4.2) - (tag: 0.4.1) - (tag: 0.4.0) - (tag: 0.3.2) - (tag: 0.3.1) - (tag: 0.3.0) - (tag: 0.2.5) - (tag: 0.2.4) - (tag: 0.2.3) - (tag: 0.2.2) - (tag: 0.2.1) - (tag: 0.2.0) - - (origin/feature/keystone-v3, gerrit/feature/keystone-v3) - (tag: 0.1.3) - (tag: 0.1.2) - (tag: 0.1.1) - (tag: 0.1.0) - (tag: folsom-1) - (tag: essex-rc1) - (tag: essex-4) - (tag: essex-3) - """) - with mock.patch('reno.utils.check_output') as co: - co.return_value = tag_list_output - actual = scanner._get_version_tags_on_branch('reporoot', - branch=None) - self.assertEqual(self.EXPECTED, actual) - - def test_keystoneclient_rhel_1_7_1(self): - # git 1.7.1 as it produces output on RHEL 6 for python-keystoneclient - # git log --simplify-by-decoration --pretty="%d" - tag_list_output = textwrap.dedent(""" - (HEAD, origin/master, origin/HEAD, master) - (tag: 2.0.0) - (tag: 1.8.1) - (tag: 1.8.0) - (tag: 1.7.1) - (tag: 1.7.0) - (tag: 1.6.0) - (tag: 1.5.0) - (tag: 1.4.0) - (tag: 1.3.0) - (tag: 1.2.0) - (tag: 1.1.0) - (tag: 1.0.0) - (tag: 0.11.2) - (tag: 0.11.1) - (tag: 0.11.0) - (tag: 0.10.1) - (tag: 0.10.0) - (tag: 0.9.0) - (tag: 0.8.0) - (tag: 0.7.1) - (tag: 0.7.0) - (tag: 0.6.0) - (tag: 0.5.1) - (tag: 0.5.0) - (tag: 0.4.2) - (tag: 0.4.1) - (tag: 0.4.0) - (tag: 0.3.2) - (tag: 0.3.1) - (tag: 0.3.0) - (tag: 0.2.5) - (tag: 0.2.4) - (tag: 0.2.3) - (tag: 0.2.2) - (tag: 0.2.1) - (tag: 0.2.0) - (tag: 0.1.3) - (0.1.2) - (tag: 0.1.1) - (0.1.0) - (tag: folsom-1) - (tag: essex-rc1) - (essex-4) - (essex-3) - """) - with mock.patch('reno.utils.check_output') as co: - co.return_value = tag_list_output - actual = scanner._get_version_tags_on_branch('reporoot', - branch=None) - self.assertEqual(self.EXPECTED, actual) + def test_tags_without_count(self): + self.scanner = scanner.Scanner(self.c) + results = self.scanner._get_tags_on_branch(None) + self.assertEqual( + ['3.0.0', '2.0.0', '1.0.0'], + results, + ) class VersionTest(Base): diff --git a/requirements.txt b/requirements.txt index ba446a2..7e2b9e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ pbr<2.0,>=1.4 Babel>=1.3 PyYAML>=3.1.0 six>=1.9.0 +dulwich>=0.15.0 # Apache-2.0