openstack/releases
Code Issues Proposed changes

report warnings when the supported versions of dependencies change

Browse Source 
Compute the dependency set for the python code being released and
report when the old minimum version no longer falls within the
specified range. For releases from master treat the message as an
error. For other branches treat the message as a warning.

Update clone_repo() to return the location where the clone was written
as a convenience to the caller.

Extract the logic for determining if a version is using
pre-versioning (alpha, beta, etc.) so it can be reused.

Change-Id: I22a2f6df7f3502e4fcbf2d61ef5fee849ab15529
Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann 2017-11-29 12:04:41 -05:00
parent 27fad96c04
commit 6ac14737b7
5 changed files with 413 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
openstack_releases/cmds/validate.py
View File

@ -44,6 +44,7 @@ from openstack_releases import npmutils
from openstack_releases import project_config
from openstack_releases import puppetutils
from openstack_releases import pythonutils
from openstack_releases import requirements
from openstack_releases import versionutils
from openstack_releases import yamlutils
@ -359,6 +360,8 @@ _TYPE_TO_RELEASE_TYPE = {
'horizon-plugin': 'horizon',
}
_PYTHON_RELEASE_TYPES = ['python-service', 'python-pypi', 'neutron', 'horizon']
def get_release_type(deliverable_info, project, workdir):
"""Return tuple with release type and boolean indicating whether it
@ -595,6 +598,26 @@ def validate_releases(deliverable_info, zuul_projects,
)
)
# If we know the previous version and the
# project is a python deliverable make sure
# the requirements haven't changed in a way
# not reflecting the version.
if prev_version and release_type in _PYTHON_RELEASE_TYPES:
# For the master branch, enforce the
# rules. For other branches just warn if
# the rules are broken because there are
# cases where we do need to support point
# releases with requirements updates.
if series_name == defaults.RELEASE:
report = mk_error
else:
report = mk_warning
requirements.find_bad_lower_bound_increases(
workdir, project['repo'],
prev_version, release['version'], project['hash'],
report,
)
for e in versionutils.validate_version(
release['version'],
release_type=release_type,

2
openstack_releases/gitutils.py
View File

@ -108,6 +108,8 @@ def clone_repo(workdir, repo, ref=None, branch=None):
cmd.append(repo)
LOG.info(' '.join(cmd))
subprocess.check_call(cmd)
dest = os.path.join(workdir, repo)
return dest
def safe_clone_repo(workdir, repo, ref, mk_error):

154
openstack_releases/requirements.py Normal file
View File

@ -0,0 +1,154 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Tools for working with requirements lists.
"""
import logging
import os.path
import subprocess
import pkg_resources
from openstack_releases import gitutils
from openstack_releases import pythonutils
from openstack_releases import versionutils
LOG = logging.getLogger(__name__)
def find_bad_lower_bound_increases(workdir, repo,
start_version, new_version, hash,
report):
if (new_version.split('.')[-1] == '0' or
versionutils.looks_like_preversion(new_version)):
# There is no need to look at the requirements so don't do the
# extra work.
return
start_reqs = get_requirements_at_ref(workdir, repo, start_version)
hash_reqs = get_requirements_at_ref(workdir, repo, hash)
compare_lower_bounds(start_reqs, hash_reqs, report)
def compare_lower_bounds(start_reqs, hash_reqs, report):
for (section, name), req in sorted(hash_reqs.items()):
if section:
display_name = '[{}]{}'.format(section, name)
else:
display_name = name
if (section, name) not in start_reqs:
report('New dependency {}'.format(display_name))
else:
old_req = start_reqs[(section, name)]
old_min = get_min_specifier(old_req.specifier)
new_min = get_min_specifier(req.specifier)
if old_min is None and new_min is None:
# No minimums are specified.
continue
if old_min is None:
# There is no minimum on the old dependency but there
# is now a new minimum.
report(('Added minimum version for dependency {} of {} '
'without at least incrementing minor number').format(
display_name, new_min))
continue
if new_min is None:
# There was a minimum but it has been removed.
continue
if old_min.version not in req.specifier:
# The old minimum is not in the new spec.
report(('Changed supported versions for dependency {} from {} '
'to {} without at least incrementing minor number').format(
display_name, old_req.specifier, req.specifier))
def get_min_specifier(specifier_set):
"Find the specifier in the set that controls the lower bound."
for s in specifier_set:
if '>' in s.operator:
return s
def get_requirements_at_ref(workdir, repo, ref):
"Check out the repo at the ref and load the list of requirements."
dest = gitutils.clone_repo(workdir, repo, ref=ref)
subprocess.check_call(['python', 'setup.py', 'sdist'], cwd=dest)
sdist_name = pythonutils.get_sdist_name(workdir, repo)
requirements_filename = os.path.join(
dest, sdist_name + '.egg-info', 'requires.txt',
)
if os.path.exists(requirements_filename):
with open(requirements_filename, 'r') as f:
body = f.read()
else:
# The package has no dependencies.
body = ''
return parse_requirements(body)
def parse_requirements(body):
"""Given the requires.txt file for an sdist, parse it.
Returns a dictionary mapping (section, pkg name) to the
requirements specifier.
Parses files that look like:
pbr>=1.6
keyring==7.3
requests>=2.5.2
PyYAML>=3.1.0
yamlordereddictloader
prompt_toolkit
tqdm
packaging>=15.2
mwclient==0.8.1
jsonschema>=2.6.0
Jinja2>=2.6
parawrap
reno>=2.0.0
sphinx>=1.6.2
pyfiglet>=0.7.5
[sphinxext]
sphinx<1.6.1,>=1.5.1
oslosphinx
sphinxcontrib.datatemplates
icalendar
"""
requirements = {}
section = ''
for line in body.splitlines():
# Ignore blank lines and comments
if (not line.strip()) or line.startswith('#'):
continue
if line.startswith('['):
section = line.lstrip('[').rstrip(']')
continue
try:
parsed_req = pkg_resources.Requirement.parse(line)
except ValueError:
LOG.warning('failed to parse %r', line)
else:
requirements[(section, parsed_req.name)] = parsed_req
return requirements

223
openstack_releases/tests/test_requirements.py Normal file
View File

@ -0,0 +1,223 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import textwrap
import mock
from oslotest import base
import pkg_resources
from openstack_releases import requirements
class TestParseRequirements(base.BaseTestCase):
def test_empty(self):
self.assertEqual({}, requirements.parse_requirements(''))
self.assertEqual({}, requirements.parse_requirements('\n'))
def test_comments(self):
self.assertEqual({}, requirements.parse_requirements('#'))
self.assertEqual({}, requirements.parse_requirements('#\n'))
def test_simple(self):
self.assertEqual(
[('', 'pbr')],
list(requirements.parse_requirements(textwrap.dedent('''
pbr>=1.6
''')).keys()),
)
def test_multiline(self):
self.assertEqual(
[('', 'keyring'),
('', 'pbr')],
list(
sorted(
requirements.parse_requirements(
textwrap.dedent('''
pbr>=1.6
keyring==7.3
''')
).keys()
)
)
)
class TestGetMinSpecifier(base.BaseTestCase):
def test_none(self):
actual = requirements.get_min_specifier(
pkg_resources.Requirement.parse('pbr').specifier,
)
self.assertIsNone(actual)
def test_greater(self):
actual = requirements.get_min_specifier(
pkg_resources.Requirement.parse('pbr>1.6').specifier,
)
self.assertEqual('1.6', actual.version)
def test_greater_equal(self):
actual = requirements.get_min_specifier(
pkg_resources.Requirement.parse('pbr>=1.6').specifier,
)
self.assertEqual('1.6', actual.version)
class TestCompareLowerBounds(base.BaseTestCase):
def test_new_requirement(self):
warnings = []
old = {
}
new = {
(None, 'pbr'): pkg_resources.Requirement.parse('pbr'),
}
requirements.compare_lower_bounds(
old, new, warnings.append,
)
print(warnings)
self.assertEqual(1, len(warnings))
def test_dropped_requirement(self):
warnings = []
old = {
(None, 'pbr'): pkg_resources.Requirement.parse('pbr'),
}
new = {
}
requirements.compare_lower_bounds(
old, new, warnings.append,
)
print(warnings)
self.assertEqual(0, len(warnings))
def test_no_lower(self):
warnings = []
old = {
(None, 'pbr'): pkg_resources.Requirement.parse('pbr'),
}
new = {
(None, 'pbr'): pkg_resources.Requirement.parse('pbr'),
}
requirements.compare_lower_bounds(
old, new, warnings.append,
)
print(warnings)
self.assertEqual(0, len(warnings))
def test_new_lower(self):
warnings = []
old = {
(None, 'pbr'): pkg_resources.Requirement.parse('pbr'),
}
new = {
(None, 'pbr'): pkg_resources.Requirement.parse('pbr>=1.6'),
}
requirements.compare_lower_bounds(
old, new, warnings.append,
)
print(warnings)
self.assertEqual(1, len(warnings))
def test_raised_lower(self):
warnings = []
old = {
(None, 'pbr'): pkg_resources.Requirement.parse('pbr>=1.5'),
}
new = {
(None, 'pbr'): pkg_resources.Requirement.parse('pbr>=1.6'),
}
requirements.compare_lower_bounds(
old, new, warnings.append,
)
print(warnings)
self.assertEqual(1, len(warnings))
def test_new_lower_format(self):
warnings = []
old = {
(None, 'pbr'): pkg_resources.Requirement.parse('pbr>=1.6'),
}
new = {
(None, 'pbr'): pkg_resources.Requirement.parse('pbr>=1.6.0'),
}
requirements.compare_lower_bounds(
old, new, warnings.append,
)
print(warnings)
self.assertEqual(0, len(warnings))
def test_new_lower_comparator(self):
warnings = []
old = {
(None, 'pbr'): pkg_resources.Requirement.parse('pbr>=1.6'),
}
new = {
(None, 'pbr'): pkg_resources.Requirement.parse('pbr>1.6'),
}
requirements.compare_lower_bounds(
old, new, warnings.append,
)
print(warnings)
self.assertEqual(1, len(warnings))
class TestFindBadLowerBoundsIncreases(base.BaseTestCase):
@mock.patch('openstack_releases.requirements.get_requirements_at_ref')
def test_skip_for_beta(self, get_req):
warnings = []
get_req.side_effect = AssertionError('should not be called')
requirements.find_bad_lower_bound_increases(
None, None, '1.0.0', '2.0.0.0b1', None,
warnings.append,
)
print(warnings)
self.assertEqual(0, len(warnings))
@mock.patch('openstack_releases.requirements.get_requirements_at_ref')
def test_skip_for_rc(self, get_req):
warnings = []
get_req.side_effect = AssertionError('should not be called')
requirements.find_bad_lower_bound_increases(
None, None, '1.0.0', '2.0.0.0rc1', None,
warnings.append,
)
print(warnings)
self.assertEqual(0, len(warnings))
@mock.patch('openstack_releases.requirements.get_requirements_at_ref')
def test_skip_for_zero_patch_major(self, get_req):
warnings = []
get_req.side_effect = AssertionError('should not be called')
requirements.find_bad_lower_bound_increases(
None, None, '1.0.0', '2.0.0', None,
warnings.append,
)
print(warnings)
self.assertEqual(0, len(warnings))
@mock.patch('openstack_releases.requirements.get_requirements_at_ref')
def test_skip_for_zero_patch_minor(self, get_req):
warnings = []
get_req.side_effect = AssertionError('should not be called')
requirements.find_bad_lower_bound_increases(
None, None, '1.0.0', '1.1.0', None,
warnings.append,
)
print(warnings)
self.assertEqual(0, len(warnings))

16
openstack_releases/versionutils.py
View File

@ -48,11 +48,9 @@ def validate_version(versionstr, release_type='python-service', pre_ok=True):
Apply our SemVer rules to version strings and report all issues.
"""
if not pre_ok:
for pre_indicator in ['a', 'b', 'rc']:
if pre_indicator in versionstr:
yield('Version %s looks like a pre-release and the release '
'model does not allow for it' % versionstr)
if not pre_ok and looks_like_preversion(versionstr):
yield('Version %s looks like a pre-release and the release '
'model does not allow for it' % versionstr)
if release_type not in _VALIDATORS:
yield 'Release Type %r not valid using \'python-service\' instead' % release_type
@ -77,3 +75,11 @@ def canonical_version(versionstr, release_type='python-service'):
if errors:
raise ValueError(errors[-1])
return versionstr
def looks_like_preversion(versionstr):
"Return boolean indicating if the version appears to be a pre-version."
for pre_indicator in ['a', 'b', 'rc']:
if pre_indicator in versionstr:
return True
return False