8a4fa891e7
Now that we allow projects to diverge in their lower bounds, it no longer makes sense for us to track "global" lower bounds values. We need to retain the exclusion specifiers (using the "!=" operator) so that the automatic constraint update calculation continues to produce a set of values that we expect to work. In addition to updating the data, this patch updates the validation rules so that if a lower bound is included an error is reported. Change-Id: If9da4636755fc089b2ec5f075e03767f84c34a67 Signed-off-by: Doug Hellmann <doug@doughellmann.com>
221 lines
7.9 KiB
Python
221 lines
7.9 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""Requirements handling."""
|
|
|
|
# This module has no IO at all, and none should be added.
|
|
|
|
import collections
|
|
import distutils.version
|
|
import packaging.specifiers
|
|
import pkg_resources
|
|
import re
|
|
|
|
|
|
# A header for the requirements file(s).
|
|
# TODO(lifeless): Remove this once constraints are in use.
|
|
_REQS_HEADER = [
|
|
'# The order of packages is significant, because pip processes '
|
|
'them in the order\n',
|
|
'# of appearance. Changing the order has an impact on the overall '
|
|
'integration\n',
|
|
'# process, which may cause wedges in the gate later.\n',
|
|
]
|
|
|
|
|
|
def key_specifier(a):
|
|
weight = {'>=': 0, '>': 0,
|
|
'===': 1, '==': 1, '~=': 1, '!=': 1,
|
|
'<': 2, '<=': 2}
|
|
a = a._spec
|
|
return (weight[a[0]], distutils.version.LooseVersion(a[1]))
|
|
|
|
|
|
class Requirement(collections.namedtuple('Requirement',
|
|
['package', 'location', 'specifiers',
|
|
'markers', 'comment', 'extras'])):
|
|
def __new__(cls, package, location, specifiers, markers, comment,
|
|
extras=None):
|
|
return super(Requirement, cls).__new__(
|
|
cls, package, location, specifiers, markers, comment,
|
|
frozenset(extras or ()))
|
|
|
|
def to_line(self, marker_sep=';', line_prefix='', comment_prefix=' ',
|
|
sort_specifiers=False):
|
|
comment_p = comment_prefix if self.package else ''
|
|
comment = (comment_p + self.comment if self.comment else '')
|
|
marker = marker_sep + self.markers if self.markers else ''
|
|
package = line_prefix + self.package if self.package else ''
|
|
location = self.location + '#egg=' if self.location else ''
|
|
extras = '[%s]' % ",".join(sorted(self.extras)) if self.extras else ''
|
|
specifiers = self.specifiers
|
|
if sort_specifiers:
|
|
_specifiers = packaging.specifiers.SpecifierSet(specifiers)
|
|
_specifiers = ['%s' % s for s in sorted(_specifiers,
|
|
key=key_specifier)]
|
|
specifiers = ','.join(_specifiers)
|
|
return '%s%s%s%s%s%s\n' % (location,
|
|
package,
|
|
extras,
|
|
specifiers,
|
|
marker,
|
|
comment)
|
|
|
|
|
|
Requirements = collections.namedtuple('Requirements', ['reqs'])
|
|
|
|
|
|
url_re = re.compile(
|
|
'^(?P<url>\s*(?:-e\s)?\s*(?:(?:[a-z]+\+)?(?:[a-z]+))://[^#]*)'
|
|
'#egg=(?P<name>[-\.\w]+)')
|
|
|
|
|
|
def canonical_name(req_name):
|
|
"""Return the canonical form of req_name."""
|
|
return pkg_resources.safe_name(req_name).lower()
|
|
|
|
|
|
def parse(content, permit_urls=False):
|
|
return to_dict(to_reqs(content, permit_urls=permit_urls))
|
|
|
|
|
|
def parse_line(req_line, permit_urls=False):
|
|
"""Parse a single line of a requirements file.
|
|
|
|
requirements files here are a subset of pip requirements files: we don't
|
|
try to parse URL entries, or pip options like -f and -e. Those are not
|
|
permitted in global-requirements.txt. If encountered in a synchronised
|
|
file such as requirements.txt or test-requirements.txt, they are illegal
|
|
but currently preserved as-is.
|
|
|
|
They may of course be used by local test configurations, just not
|
|
committed into the OpenStack reference branches.
|
|
|
|
:param permit_urls: If True, urls are parsed into Requirement tuples.
|
|
By default they are not, because they cannot be reflected into
|
|
setuptools kwargs, and thus the default is conservative. When
|
|
urls are permitted, -e *may* be supplied at the start of the line.
|
|
"""
|
|
end = len(req_line)
|
|
hash_pos = req_line.find('#')
|
|
if hash_pos < 0:
|
|
hash_pos = end
|
|
# Don't find urls that are in comments.
|
|
if '://' in req_line[:hash_pos]:
|
|
if permit_urls:
|
|
# We accept only a subset of urls here - they have to have an egg
|
|
# name so that we can tell what project its for without doing
|
|
# network access. Egg markers use a fragment, so we need to pull
|
|
# out url from the entire line.
|
|
m = url_re.match(req_line)
|
|
name = m.group('name')
|
|
location = m.group('url')
|
|
parse_start = m.end('name')
|
|
hash_pos = req_line[parse_start:].find('#')
|
|
if hash_pos < 0:
|
|
hash_pos = end
|
|
else:
|
|
hash_pos = hash_pos + parse_start
|
|
else:
|
|
# Trigger an early failure before we look for ':'
|
|
pkg_resources.Requirement.parse(req_line)
|
|
else:
|
|
parse_start = 0
|
|
location = ''
|
|
semi_pos = req_line.find(';', parse_start, hash_pos)
|
|
colon_pos = req_line.find(':', parse_start, hash_pos)
|
|
marker_pos = max(semi_pos, colon_pos)
|
|
if marker_pos < 0:
|
|
marker_pos = hash_pos
|
|
markers = req_line[marker_pos + 1:hash_pos].strip()
|
|
if hash_pos != end:
|
|
comment = req_line[hash_pos:]
|
|
else:
|
|
comment = ''
|
|
req_line = req_line[parse_start:marker_pos]
|
|
|
|
extras = ()
|
|
if parse_start:
|
|
# We parsed a url before
|
|
specifier = ''
|
|
elif req_line:
|
|
# Pulled out a requirement
|
|
parsed = pkg_resources.Requirement.parse(req_line)
|
|
name = parsed.project_name
|
|
extras = parsed.extras
|
|
specifier = str(parsed.specifier)
|
|
else:
|
|
# Comments / blank lines etc.
|
|
name = ''
|
|
specifier = ''
|
|
return Requirement(name, location, specifier, markers, comment, extras)
|
|
|
|
|
|
def to_content(reqs, marker_sep=';', line_prefix='', prefix=True):
|
|
lines = []
|
|
if prefix:
|
|
lines += _REQS_HEADER
|
|
for req in reqs.reqs:
|
|
lines.append(req.to_line(marker_sep, line_prefix))
|
|
return u''.join(lines)
|
|
|
|
|
|
def to_dict(req_sequence):
|
|
reqs = dict()
|
|
for req, req_line in req_sequence:
|
|
if req is not None:
|
|
key = canonical_name(req.package)
|
|
reqs.setdefault(key, []).append((req, req_line))
|
|
return reqs
|
|
|
|
|
|
def _pass_through(req_line, permit_urls=False):
|
|
"""Identify unparsable lines."""
|
|
if permit_urls:
|
|
return (req_line.startswith('http://tarballs.openstack.org/') or
|
|
req_line.startswith('-f'))
|
|
else:
|
|
return (req_line.startswith('http://tarballs.openstack.org/') or
|
|
req_line.startswith('-e') or
|
|
req_line.startswith('-f'))
|
|
|
|
|
|
def to_reqs(content, permit_urls=False):
|
|
for content_line in content.splitlines(True):
|
|
req_line = content_line.strip()
|
|
if _pass_through(req_line, permit_urls=permit_urls):
|
|
yield None, content_line
|
|
else:
|
|
yield parse_line(req_line, permit_urls=permit_urls), content_line
|
|
|
|
|
|
def check_reqs_bounds_policy(global_reqs):
|
|
"""Check that the global requirement version specifiers match the policy.
|
|
|
|
The policy is defined as
|
|
* There needs to be exactly one lower bound (>=1.2 defined)
|
|
* There can be one or more excludes (!=1.2.1, !=1.2.2)
|
|
* TODO: Clarify (non-) existance of upper caps
|
|
"""
|
|
|
|
for pkg_requirement in global_reqs.values():
|
|
req = pkg_requirement[0][0]
|
|
if req.package:
|
|
_specifiers = packaging.specifiers.SpecifierSet(req.specifiers)
|
|
lower_bound = set()
|
|
for spec in _specifiers:
|
|
if spec.operator == '>=':
|
|
lower_bound.add(spec)
|
|
if len(lower_bound):
|
|
yield ('Requirement %s should not include a >= specifier' %
|
|
req.package)
|