openstack/deb-python-reno
Code Issues Proposed changes

use dulwich to determine the tags on a branch

Browse Source 
Replace the git commands for determine the tags on a branch with calls
to dulwich.

Because the Scanner class scans for the tags when it is created and
caches the data, we need to update the tests to instantiate the Scanner
after the git repo is created and the test commits are applied. The main
API entry point into reno is the Loader, and that is not changed because
the Loader does not instantiate a Scanner until it knows it needs it.

Change-Id: Icf7c7a47a768175a7cb31ab374aca4fa4b44a5cb
Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann
2016-12-05 18:26:14 -05:00
parent db4e3647e2
commit 5090095169
4 changed files with 97 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
reno/loader.py
View File

@@ -52,7 +52,7 @@ class Loader(object):
self._earliest_version = conf.earliest_version
self._cache = None
self._scanner = scanner.Scanner(self._config)
self._scanner = None
self._scanner_output = None
self._cache_filename = get_cache_filename(self._reporoot,
self._notespath)
@@ -76,6 +76,7 @@ class Loader(object):
for n in self._cache['notes']
}
else:
self._scanner = scanner.Scanner(self._config)
self._scanner_output = self._scanner.get_notes_by_version()
@property

71
reno/scanner.py
View File

@@ -20,6 +20,9 @@ import re
import subprocess
import sys
from dulwich import refs
from dulwich import repo
from reno import utils
LOG = logging.getLogger(__name__)
@@ -78,40 +81,48 @@ PRE_RELEASE_RE = re.compile('''
''', flags=re.VERBOSE | re.UNICODE)
def _get_version_tags_on_branch(reporoot, branch):
"""Return tags from the branch, in date order.
Need to get the list of tags in the right order, because the topo
search breaks the date ordering. Use git to ask for the tags in
order, rather than trying to sort them, because many repositories
have "non-standard" tags or have renumbered projects (from
date-based to SemVer), for which sorting would require complex
logic.
"""
tags = []
tag_cmd = [
'git', 'log',
'--simplify-by-decoration',
'--pretty="%d"',
]
if branch:
tag_cmd.append(branch)
LOG.debug('running %s' % ' '.join(tag_cmd))
tag_results = utils.check_output(tag_cmd, cwd=reporoot)
LOG.debug(tag_results)
for line in tag_results.splitlines():
LOG.debug('line %r' % line)
for match in TAG_RE.findall(line):
tags.append(match)
return tags
class Scanner(object):
def __init__(self, conf):
self.conf = conf
self.reporoot = self.conf.reporoot
self._repo = repo.Repo(self.reporoot)
self._load_tags()
def _load_tags(self):
self._all_tags = {
k.partition(b'/tags/')[-1].decode('utf-8'): v
for k, v in self._repo.get_refs().items()
if k.startswith(b'refs/tags/')
}
self._shas_to_tags = {}
for tag, tag_sha in self._all_tags.items():
# The tag has its own SHA, but the tag refers to the commit and
# that's the SHA we'll see when we scan commits on a branch.
tag_obj = self._repo[tag_sha]
tagged_sha = tag_obj.object[1]
self._shas_to_tags.setdefault(tagged_sha, []).append(tag)
def _get_tags_on_branch(self, branch):
"Return a list of tag names on the given branch."
results = []
if branch:
branch_ref = b'refs/heads/' + branch.encode('utf-8')
if not refs.check_ref_format(branch_ref):
raise ValueError(
'{!r} does not look like a valid branch reference'.format(
branch_ref))
branch_head = self._repo.refs[branch_ref]
else:
branch_head = self._repo.refs[b'HEAD']
w = self._repo.get_walker(branch_head)
for c in w:
# shas_to_tags has encoded versions of the shas
# but the commit object gives us a decoded version
sha = c.commit.sha().hexdigest().encode('ascii')
if sha in self._shas_to_tags:
results.extend(self._shas_to_tags[sha])
return results
def _get_current_version(self, branch=None):
"""Return the current version of the repository.
@@ -252,7 +263,7 @@ class Scanner(object):
# order. We scan the commit history in topological order to ensure
# we have the commits in the right version, so we might encounter
# the tags in a different order during that phase.
versions_by_date = _get_version_tags_on_branch(reporoot, branch)
versions_by_date = self._get_tags_on_branch(branch)
LOG.debug('versions by date %r' % (versions_by_date,))
versions = []
earliest_seen = collections.OrderedDict()

215
reno/tests/test_scanner.py
View File

@@ -19,11 +19,9 @@ import logging
import os.path
import re
import subprocess
import textwrap
import unittest
import fixtures
import mock
from testtools.content import text_content
from reno import config
@@ -197,7 +195,6 @@ class Base(base.TestCase):
self.temp_dir = self.useFixture(fixtures.TempDir()).path
self.reporoot = os.path.join(self.temp_dir, 'reporoot')
self.c = config.Config(self.reporoot)
self.scanner = scanner.Scanner(self.c)
self._git_setup()
self._counter = itertools.count(1)
self.get_note_num = lambda: next(self._counter)
@@ -207,6 +204,7 @@ class BasicTest(Base):
def test_non_python_no_tags(self):
filename = self._add_notes_file()
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -220,6 +218,7 @@ class BasicTest(Base):
def test_python_no_tags(self):
self._make_python_package()
filename = self._add_notes_file()
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -234,6 +233,7 @@ class BasicTest(Base):
filename = self._add_notes_file()
self._add_other_file('not-a-release-note.txt')
self._run_git('tag', '-s', '-m', 'first tag', '1.0.0')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -247,6 +247,7 @@ class BasicTest(Base):
def test_note_commit_tagged(self):
filename = self._add_notes_file()
self._run_git('tag', '-s', '-m', 'first tag', '1.0.0')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -261,6 +262,7 @@ class BasicTest(Base):
self._make_python_package()
self._run_git('tag', '-s', '-m', 'first tag', '1.0.0')
filename = self._add_notes_file()
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -276,6 +278,7 @@ class BasicTest(Base):
self._add_other_file('ignore-1.txt')
self._run_git('tag', '-s', '-m', 'first tag', '1.0.0')
self._add_other_file('ignore-2.txt')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -291,6 +294,7 @@ class BasicTest(Base):
self._run_git('tag', '-s', '-m', 'first tag', '1.0.0')
f1 = self._add_notes_file()
f2 = self._add_notes_file()
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -306,6 +310,7 @@ class BasicTest(Base):
f1 = self._add_notes_file(commit=False)
f2 = self._add_notes_file()
self._run_git('tag', '-s', '-m', 'first tag', '1.0.0')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -322,6 +327,7 @@ class BasicTest(Base):
f1 = self._add_notes_file()
self._run_git('tag', '-s', '-m', 'first tag', '2.0.0')
f2 = self._add_notes_file()
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -342,6 +348,7 @@ class BasicTest(Base):
f2 = f1.replace('slug1', 'slug2')
self._run_git('mv', f1, f2)
self._git_commit('rename note file')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -361,6 +368,7 @@ class BasicTest(Base):
f2 = f1.replace('slug1', 'slug0')
self._run_git('mv', f1, f2)
self._git_commit('rename note file')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -380,6 +388,7 @@ class BasicTest(Base):
with open(os.path.join(self.reporoot, f1), 'w') as f:
f.write('---\npreamble: new contents for file')
self._git_commit('edit note file')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -399,6 +408,7 @@ class BasicTest(Base):
f2 = f1.replace('slug1', 'slug2')
self._run_git('mv', f1, f2)
self._git_commit('rename note file')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -421,6 +431,7 @@ class BasicTest(Base):
'slug1-0000000000000001')
self._run_git('mv', f1, f2)
self._git_commit('rename note file')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -443,6 +454,7 @@ class BasicTest(Base):
self.c.override(
earliest_version='2.0.0',
)
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -463,6 +475,7 @@ class BasicTest(Base):
self._run_git('rm', f1)
self._git_commit('remove note file')
self._run_git('tag', '-s', '-m', 'first tag', '2.0.0')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -493,6 +506,7 @@ class BasicTest(Base):
'--pretty=%H %d',
'--name-only')
self.addDetail('git log', text_content(log_results))
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -512,6 +526,7 @@ class PreReleaseTest(Base):
self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a1')
f1 = self._add_notes_file('slug1')
self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0a2')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -528,6 +543,7 @@ class PreReleaseTest(Base):
self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b1')
f1 = self._add_notes_file('slug1')
self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0b2')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -544,6 +560,7 @@ class PreReleaseTest(Base):
self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc1')
f1 = self._add_notes_file('slug1')
self._run_git('tag', '-s', '-m', 'first tag', '1.0.0.0rc2')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -569,6 +586,7 @@ class PreReleaseTest(Base):
self.c.override(
collapse_pre_releases=True,
)
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -591,6 +609,7 @@ class PreReleaseTest(Base):
self.c.override(
collapse_pre_releases=True,
)
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -616,6 +635,7 @@ class PreReleaseTest(Base):
self.c.override(
collapse_pre_releases=True,
)
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -644,6 +664,7 @@ class MergeCommitTest(Base):
self._run_git('merge', '--no-ff', 'test_merge_commit')
self._add_other_file('ignore-2.txt')
self._run_git('tag', '-s', '-m', 'second tag', '2.0.0')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -672,6 +693,7 @@ class MergeCommitTest(Base):
self._run_git('merge', '--no-ff', 'test_merge_commit')
self._add_other_file('ignore-2.txt')
self._run_git('tag', '-s', '-m', 'second tag', '2.0.0')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -704,6 +726,7 @@ class MergeCommitTest(Base):
self._add_other_file('ignore-2.txt')
self._run_git('tag', '-s', '-m', 'third tag', '2.0.0')
self._add_other_file('ignore-3.txt')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -740,6 +763,7 @@ class MergeCommitTest(Base):
self._add_other_file('ignore-2.txt')
self._run_git('tag', '-s', '-m', 'third tag', '2.0.0')
self._add_other_file('ignore-3.txt')
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -788,8 +812,9 @@ class BranchTest(Base):
self._run_git('checkout', '2.0.0')
self._run_git('checkout', '-b', 'stable/2')
f21 = self._add_notes_file('slug21')
log_text = self._run_git('log')
log_text = self._run_git('log', '--decorate')
self.addDetail('git log', text_content(log_text))
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -815,6 +840,7 @@ class BranchTest(Base):
self.c.override(
branch='stable/2',
)
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -842,6 +868,7 @@ class BranchTest(Base):
self.c.override(
stop_at_branch_base=False,
)
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -878,6 +905,7 @@ class BranchTest(Base):
branch='stable/4',
collapse_pre_releases=False,
)
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -914,6 +942,7 @@ class BranchTest(Base):
branch='stable/4',
collapse_pre_releases=True,
)
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -947,6 +976,7 @@ class BranchTest(Base):
self.c.override(
branch='stable/4',
)
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -980,6 +1010,7 @@ class BranchTest(Base):
self.c.override(
branch='stable/4',
)
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -1011,6 +1042,7 @@ class BranchTest(Base):
self.c.override(
branch='stable/4',
)
self.scanner = scanner.Scanner(self.c)
raw_results = self.scanner.get_notes_by_version()
results = {
k: [f for (f, n) in v]
@@ -1024,166 +1056,25 @@ class BranchTest(Base):
)
class GetTagsParseTest(base.TestCase):
class TagsTest(Base):
EXPECTED = [
'2.0.0',
'1.8.1',
'1.8.0',
'1.7.1',
'1.7.0',
'1.6.0',
'1.5.0',
'1.4.0',
'1.3.0',
'1.2.0',
'1.1.0',
'1.0.0',
'0.11.2',
'0.11.1',
'0.11.0',
'0.10.1',
'0.10.0',
'0.9.0',
'0.8.0',
'0.7.1',
'0.7.0',
'0.6.0',
'0.5.1',
'0.5.0',
'0.4.2',
'0.4.1',
'0.4.0',
'0.3.2',
'0.3.1',
'0.3.0',
'0.2.5',
'0.2.4',
'0.2.3',
'0.2.2',
'0.2.1',
'0.2.0',
'0.1.3',
'0.1.2',
'0.1.1',
'0.1.0',
]
def setUp(self):
super(TagsTest, self).setUp()
self._make_python_package()
self.f1 = self._add_notes_file('slug1')
self._run_git('tag', '-s', '-m', 'first tag', '1.0.0')
self.f2 = self._add_notes_file('slug2')
self._run_git('tag', '-s', '-m', 'first tag', '2.0.0')
self._add_notes_file('slug3')
self._run_git('tag', '-s', '-m', 'first tag', '3.0.0')
def test_keystoneclient_ubuntu_1_9_1(self):
# git 1.9.1 as it produces output on ubuntu for python-keystoneclient
# git log --simplify-by-decoration --pretty="%d"
tag_list_output = textwrap.dedent("""
(HEAD, origin/master, origin/HEAD, gerrit/master, master)
(apu/master)
(tag: 2.0.0)
(tag: 1.8.1)
(tag: 1.8.0)
(tag: 1.7.1)
(tag: 1.7.0)
(tag: 1.6.0)
(tag: 1.5.0)
(tag: 1.4.0)
(uncap-requirements)
(tag: 1.3.0)
(tag: 1.2.0)
(tag: 1.1.0)
(tag: 1.0.0)
(tag: 0.11.2)
(tag: 0.11.1)
(tag: 0.11.0)
(tag: 0.10.1)
(tag: 0.10.0)
(tag: 0.9.0)
(tag: 0.8.0)
(tag: 0.7.1)
(tag: 0.7.0)
(tag: 0.6.0)
(tag: 0.5.1)
(tag: 0.5.0)
(tag: 0.4.2)
(tag: 0.4.1)
(tag: 0.4.0)
(tag: 0.3.2)
(tag: 0.3.1)
(tag: 0.3.0)
(tag: 0.2.5)
(tag: 0.2.4)
(tag: 0.2.3)
(tag: 0.2.2)
(tag: 0.2.1)
(tag: 0.2.0)
(origin/feature/keystone-v3, gerrit/feature/keystone-v3)
(tag: 0.1.3)
(tag: 0.1.2)
(tag: 0.1.1)
(tag: 0.1.0)
(tag: folsom-1)
(tag: essex-rc1)
(tag: essex-4)
(tag: essex-3)
""")
with mock.patch('reno.utils.check_output') as co:
co.return_value = tag_list_output
actual = scanner._get_version_tags_on_branch('reporoot',
branch=None)
self.assertEqual(self.EXPECTED, actual)
def test_keystoneclient_rhel_1_7_1(self):
# git 1.7.1 as it produces output on RHEL 6 for python-keystoneclient
# git log --simplify-by-decoration --pretty="%d"
tag_list_output = textwrap.dedent("""
(HEAD, origin/master, origin/HEAD, master)
(tag: 2.0.0)
(tag: 1.8.1)
(tag: 1.8.0)
(tag: 1.7.1)
(tag: 1.7.0)
(tag: 1.6.0)
(tag: 1.5.0)
(tag: 1.4.0)
(tag: 1.3.0)
(tag: 1.2.0)
(tag: 1.1.0)
(tag: 1.0.0)
(tag: 0.11.2)
(tag: 0.11.1)
(tag: 0.11.0)
(tag: 0.10.1)
(tag: 0.10.0)
(tag: 0.9.0)
(tag: 0.8.0)
(tag: 0.7.1)
(tag: 0.7.0)
(tag: 0.6.0)
(tag: 0.5.1)
(tag: 0.5.0)
(tag: 0.4.2)
(tag: 0.4.1)
(tag: 0.4.0)
(tag: 0.3.2)
(tag: 0.3.1)
(tag: 0.3.0)
(tag: 0.2.5)
(tag: 0.2.4)
(tag: 0.2.3)
(tag: 0.2.2)
(tag: 0.2.1)
(tag: 0.2.0)
(tag: 0.1.3)
(0.1.2)
(tag: 0.1.1)
(0.1.0)
(tag: folsom-1)
(tag: essex-rc1)
(essex-4)
(essex-3)
""")
with mock.patch('reno.utils.check_output') as co:
co.return_value = tag_list_output
actual = scanner._get_version_tags_on_branch('reporoot',
branch=None)
self.assertEqual(self.EXPECTED, actual)
def test_tags_without_count(self):
self.scanner = scanner.Scanner(self.c)
results = self.scanner._get_tags_on_branch(None)
self.assertEqual(
['3.0.0', '2.0.0', '1.0.0'],
results,
)
class VersionTest(Base):

1
requirements.txt
View File

@@ -6,3 +6,4 @@ pbr<2.0,>=1.4
Babel>=1.3
PyYAML>=3.1.0
six>=1.9.0
dulwich>=0.15.0 # Apache-2.0