d39eed2be7
Using upper limits for all requirements adds a lot of pain for operators and packaging teams. Sometimes it is very hard to find a list of compatible packages which will feet rally and some other libs. This patch stops setting upper limits for all packages in our requirements list and moves to our personal upper-constraints file. PS: It doesn't mean that we cannot limit packages in case of failures and known compatibility issues Change-Id: Id5d84fd1b605811ecbf7f88a4af8c2c607b9dd72
386 lines
14 KiB
Python
386 lines
14 KiB
Python
#!/usr/bin/env python
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""
|
|
Synchronizes, formats and prepares requirements to release(obtains and adds
|
|
maximum allowed version).
|
|
"""
|
|
|
|
import collections
|
|
import logging
|
|
import re
|
|
import sys
|
|
import textwrap
|
|
|
|
import requests
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
if not LOG.handlers:
|
|
LOG.addHandler(logging.StreamHandler())
|
|
LOG.setLevel(logging.INFO)
|
|
|
|
|
|
GLOBAL_REQUIREMENTS_LOCATIONS = (
|
|
"https://raw.githubusercontent.com/openstack/requirements/master/",
|
|
"http://git.openstack.org/cgit/openstack/requirements/plain/"
|
|
)
|
|
RALLY_REQUIREMENTS_FILES = (
|
|
"requirements.txt",
|
|
"test-requirements.txt"
|
|
)
|
|
DO_NOT_TOUCH_TAG = "[do-not-touch]"
|
|
|
|
|
|
class Comment(object):
|
|
def __init__(self, s=None, finished=False):
|
|
self._comments = []
|
|
self.is_finished = finished
|
|
if s:
|
|
self.append(s)
|
|
|
|
def finish_him(self):
|
|
self.is_finished = True
|
|
|
|
def append(self, s):
|
|
self._comments.append(s[1:].strip())
|
|
|
|
def __str__(self):
|
|
return textwrap.fill("\n".join(self._comments), width=80,
|
|
initial_indent="# ", subsequent_indent="# ")
|
|
|
|
|
|
_PYPI_CACHE = {}
|
|
|
|
|
|
class PYPIPackage(object):
|
|
# NOTE(andreykurilin): one license can have different labels. Let's use
|
|
# unified variant.
|
|
LICENSE_MAP = {"MIT license": "MIT",
|
|
"MIT License": "MIT",
|
|
"BSD License": "BSD",
|
|
"Apache 2.0": "Apache License, Version 2.0"}
|
|
|
|
def __init__(self, package_name):
|
|
self.package_name = package_name
|
|
self._pypi_info = None
|
|
self._pypi_license = None
|
|
|
|
@property
|
|
def pypi_info(self):
|
|
if self._pypi_info is None:
|
|
if self.package_name in _PYPI_CACHE:
|
|
self._pypi_info = _PYPI_CACHE[self.package_name]
|
|
else:
|
|
resp = requests.get("https://pypi.python.org/pypi/%s/json" %
|
|
self.package_name)
|
|
if resp.status_code != 200:
|
|
print("An error occurred while checking '%s' package at "
|
|
"pypi." % self.package_name)
|
|
raise Exception(resp.text)
|
|
self._pypi_info = resp.json()
|
|
|
|
# let's cache it for the case when we need to sync requirements
|
|
# and update upper constrains
|
|
_PYPI_CACHE[self.package_name] = self._pypi_info
|
|
return self._pypi_info
|
|
|
|
@property
|
|
def pypi_version(self):
|
|
return self.pypi_info["info"]["version"]
|
|
|
|
@property
|
|
def pypi_license(self):
|
|
if self._pypi_license is None:
|
|
if self.pypi_info["info"]["license"]:
|
|
self._pypi_license = self.pypi_info["info"]["license"]
|
|
else:
|
|
# try to parse classifiers
|
|
prefix = "License :: OSI Approved :: "
|
|
classifiers = [c[len(prefix):]
|
|
for c in self.pypi_info["info"]["classifiers"]
|
|
if c.startswith(prefix)]
|
|
self._pypi_license = "/".join(classifiers)
|
|
self._license = self.LICENSE_MAP.get(self._pypi_license,
|
|
self._pypi_license)
|
|
if self._pypi_license == "UNKNOWN":
|
|
self._pypi_license = None
|
|
return self._license
|
|
|
|
def __eq__(self, other):
|
|
return (isinstance(other, PYPIPackage) and
|
|
self.package_name == other.package_name)
|
|
|
|
|
|
class Requirement(PYPIPackage):
|
|
RE_NAME = re.compile(r"[a-zA-Z0-9-._]+")
|
|
RE_CONST_VERSION = re.compile(r"==[a-zA-Z0-9.]+")
|
|
RE_MIN_VERSION = re.compile(r">=?[a-zA-Z0-9.]+")
|
|
RE_MAX_VERSION = re.compile(r"<=?[a-zA-Z0-9.]+")
|
|
RE_NE_VERSIONS = re.compile(r"!=[a-zA-Z0-9.]+")
|
|
|
|
def __init__(self, package_name, version):
|
|
super(Requirement, self).__init__(package_name)
|
|
self.version = version
|
|
self.do_not_touch = False
|
|
|
|
def sync_max_version_with_pypy(self):
|
|
if isinstance(self.version, dict) and not self.do_not_touch:
|
|
self.version["max"] = "<=%s" % self.pypi_version
|
|
|
|
@classmethod
|
|
def parse_line(cls, line):
|
|
match = cls.RE_NAME.match(line)
|
|
if match:
|
|
name = match.group()
|
|
# remove name
|
|
versions = line.replace(name, "")
|
|
# remove comments
|
|
versions = versions.split("#")[0]
|
|
# remove python classifiers
|
|
versions = versions.split(";")[0].strip()
|
|
if not cls.RE_CONST_VERSION.match(versions):
|
|
versions = versions.strip().split(",")
|
|
min_version = None
|
|
max_version = None
|
|
ne_versions = []
|
|
for version in versions:
|
|
if cls.RE_MIN_VERSION.match(version):
|
|
if min_version:
|
|
raise Exception("Found several min versions for "
|
|
"%s package." % name)
|
|
min_version = version
|
|
elif cls.RE_MAX_VERSION.match(version):
|
|
if max_version:
|
|
raise Exception("Found several max versions for "
|
|
"%s package." % name)
|
|
max_version = version
|
|
elif cls.RE_NE_VERSIONS.match(version):
|
|
ne_versions.append(version)
|
|
versions = {"min": min_version,
|
|
"max": max_version,
|
|
"ne": ne_versions}
|
|
return cls(name, versions)
|
|
|
|
def __str__(self):
|
|
if isinstance(self.version, dict):
|
|
version = []
|
|
|
|
min_equal_to_max = False
|
|
if self.version["min"] and self.version["max"]:
|
|
if (self.version["min"].startswith(">=") and
|
|
self.version["max"].startswith("<=") and
|
|
self.version["min"][2:] == self.version["max"][2:]):
|
|
# min and max versions are equal there is no need to write
|
|
# both of them
|
|
min_equal_to_max = True
|
|
version.append("==%s" % self.version["min"][2:])
|
|
|
|
if not min_equal_to_max and self.version["min"]:
|
|
version.append(self.version["min"])
|
|
|
|
if not min_equal_to_max and self.version["ne"]:
|
|
version.extend(self.version["ne"])
|
|
|
|
if not min_equal_to_max and self.version["max"]:
|
|
version.append(self.version["max"])
|
|
|
|
version = ",".join(version)
|
|
else:
|
|
if self.do_not_touch:
|
|
version = self.version
|
|
else:
|
|
# remove const version
|
|
version = ">=%s" % self.version[2:]
|
|
|
|
string = "%s%s" % (self.package_name, version)
|
|
if self.pypi_license:
|
|
# NOTE(andreykurilin): When I start implementation of this script,
|
|
# python-keystoneclient dependency string took around ~45-55
|
|
# chars, so let's use this length as indent. Feel free to modify
|
|
# it to lower or greater value.
|
|
magic_number = 55
|
|
if len(string) < magic_number:
|
|
indent = magic_number - len(string)
|
|
else:
|
|
indent = 2
|
|
string += " " * indent + "# " + self.pypi_license
|
|
return string
|
|
|
|
def __eq__(self, other):
|
|
return (isinstance(other, self.__class__) and
|
|
self.package_name == other.package_name)
|
|
|
|
def __ne__(self, other):
|
|
return not self.__eq__(other)
|
|
|
|
|
|
class UpperConstraint(PYPIPackage):
|
|
|
|
RE_LINE = re.compile(
|
|
r"(?P<package_name>[a-zA-Z0-9-._]+)===(?P<version>[a-zA-Z0-9.]+)")
|
|
|
|
def __init__(self, package_name, version=None):
|
|
super(UpperConstraint, self).__init__(package_name)
|
|
self._version = version
|
|
|
|
def __str__(self):
|
|
return "%s===%s" % (self.package_name, self.version)
|
|
|
|
@property
|
|
def version(self):
|
|
if self._version is None:
|
|
self._version = self.pypi_version
|
|
return self._version
|
|
|
|
@classmethod
|
|
def parse_line(cls, line):
|
|
match = cls.RE_LINE.match(line)
|
|
if match:
|
|
return cls(**match.groupdict())
|
|
|
|
def update(self, version):
|
|
self._version = version
|
|
|
|
|
|
def parse_data(raw_data, include_comments=True, dependency_cls=Requirement):
|
|
# first elem is None to simplify checks of last elem in requirements
|
|
requirements = [None]
|
|
for line in raw_data.split("\n"):
|
|
if line.startswith("#"):
|
|
if not include_comments:
|
|
continue
|
|
|
|
if getattr(requirements[-1], "is_finished", True):
|
|
requirements.append(Comment())
|
|
|
|
requirements[-1].append(line)
|
|
elif line == "":
|
|
# just empty line
|
|
if isinstance(requirements[-1], Comment):
|
|
requirements[-1].finish_him()
|
|
requirements.append(Comment(finished=True))
|
|
else:
|
|
if (isinstance(requirements[-1], Comment) and
|
|
not requirements[-1].is_finished):
|
|
requirements[-1].finish_him()
|
|
|
|
# parse_line
|
|
dep = dependency_cls.parse_line(line)
|
|
if dep:
|
|
if (isinstance(requirements[-1], Comment) and
|
|
DO_NOT_TOUCH_TAG in str(requirements[-1])):
|
|
dep.do_not_touch = True
|
|
requirements.append(dep)
|
|
|
|
for i in range(len(requirements) - 1, 0, -1):
|
|
# remove empty lines at the end of file
|
|
if isinstance(requirements[i], Comment):
|
|
if str(requirements[i]) == "":
|
|
requirements.pop(i)
|
|
else:
|
|
break
|
|
return collections.OrderedDict(
|
|
(v if isinstance(v, Comment) else v.package_name, v)
|
|
for v in requirements if v)
|
|
|
|
|
|
def _fetch_from_gr(filename):
|
|
"""Try to fetch data from OpenStack global-requirements repo"""
|
|
for i in range(0, len(GLOBAL_REQUIREMENTS_LOCATIONS)):
|
|
url = GLOBAL_REQUIREMENTS_LOCATIONS[i] + filename
|
|
LOG.debug("Try to obtain %s from %s" % (filename, url))
|
|
try:
|
|
return requests.get(url).text
|
|
except requests.ConnectionError as e:
|
|
LOG.exception(e)
|
|
raise Exception("Unable to obtain %s" % filename)
|
|
|
|
|
|
def _write_requirements(filename, requirements):
|
|
"""Saves requirements to file."""
|
|
if isinstance(requirements, dict):
|
|
requirements = requirements.values()
|
|
LOG.info("Saving requirements to %s." % filename)
|
|
with open(filename, "w") as f:
|
|
for entity in requirements:
|
|
f.write(str(entity))
|
|
f.write("\n")
|
|
|
|
|
|
def sync_requirements():
|
|
"""Synchronizes Rally requirements with OpenStack global-requirements."""
|
|
LOG.info("Obtaining global-requirements of OpenStack...")
|
|
raw_gr = _fetch_from_gr("global-requirements.txt")
|
|
|
|
# NOTE(andreykurilin): global-requirements includes comments which can be
|
|
# unrelated to Rally project, so let's just ignore them
|
|
gr = parse_data(raw_gr, include_comments=False)
|
|
for file_name in RALLY_REQUIREMENTS_FILES:
|
|
LOG.debug("Processing '%s'." % file_name)
|
|
with open(file_name) as f:
|
|
requirements = parse_data(f.read())
|
|
for name, req in requirements.items():
|
|
if isinstance(req, Requirement) and not req.do_not_touch:
|
|
if name in gr:
|
|
req.version = gr[req.package_name].version
|
|
else:
|
|
# it not g-r requirements
|
|
if isinstance(req.version, dict):
|
|
req.version["max"] = None
|
|
_write_requirements(file_name, requirements)
|
|
|
|
|
|
def update_upper_constraints():
|
|
"""Obtains latest version of packages and put them to upper-constraints."""
|
|
LOG.info("Obtaining upper-constrains from OpenStack...")
|
|
raw_g_uc = _fetch_from_gr("upper-constraints.txt")
|
|
# NOTE(andreykurilin): global OpenStack upper-constraints file includes
|
|
# comments which can be unrelated to Rally project, so let's just ignore
|
|
# them.
|
|
global_uc = parse_data(raw_g_uc, include_comments=False,
|
|
dependency_cls=UpperConstraint)
|
|
with open("upper-constraints.txt") as f:
|
|
our_uc = parse_data(f.read(), dependency_cls=UpperConstraint)
|
|
with open("requirements.txt") as f:
|
|
our_requirements = parse_data(f.read(), include_comments=False)
|
|
|
|
for name, req in our_requirements.items():
|
|
if isinstance(req, Comment):
|
|
print("continue")
|
|
continue
|
|
print(req)
|
|
if name not in our_uc:
|
|
our_uc[name] = UpperConstraint(name)
|
|
|
|
if name in global_uc:
|
|
# we cannot use whatever we want versions in CI. OpenStack CI
|
|
# installs ignores versions listed in requirements of
|
|
# particular project and use versions from global u-c file.
|
|
# It means that we need to suggest to use the same versions
|
|
our_uc[name].update(global_uc[name].version)
|
|
|
|
our_uc = sorted(our_uc.values(), key=lambda o: o.package_name.upper())
|
|
_write_requirements("upper-constraints.txt", our_uc)
|
|
|
|
|
|
def main():
|
|
sync_requirements()
|
|
update_upper_constraints()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|