[requirements] Stop using upper limits for all cases

Using upper limits for all requirements adds a lot of pain for operators
and packaging teams. Sometimes it is very hard to find a list of
compatible packages which will feet rally and some other libs.
This patch stops setting upper limits for all packages in our
requirements list and moves to our personal upper-constraints file.

PS: It doesn't mean that we cannot limit packages in case of failures
and known compatibility issues

Change-Id: Id5d84fd1b605811ecbf7f88a4af8c2c607b9dd72
This commit is contained in:
Andrey Kurilin 2017-11-06 15:38:49 +02:00
parent 7153e0cbc5
commit d39eed2be7
4 changed files with 267 additions and 177 deletions

View File

@ -3,50 +3,50 @@
# process, which may cause wedges in the gate later. # process, which may cause wedges in the gate later.
# Rally core dependencies # Rally core dependencies
alembic>=0.8.10,<=0.9.5 # MIT alembic>=0.8.10 # MIT
decorator>=3.4.0,<=4.1.2 # new BSD License decorator>=3.4.0 # new BSD License
Jinja2>=2.8,!=2.9.0,!=2.9.1,!=2.9.2,!=2.9.3,!=2.9.4,<=2.9.6 # BSD Jinja2>=2.8,!=2.9.0,!=2.9.1,!=2.9.2,!=2.9.3,!=2.9.4 # BSD
jsonschema>=2.0.0,!=2.5.0,<3.0.0 # MIT jsonschema>=2.6.0,<3.0.0 # MIT
morph morph # GPLv3+
netaddr>=0.7.13,!=0.7.16,<=0.7.19 # BSD netaddr>=0.7.18 # BSD
oslo.config>=4.0.0,!=4.3.0,!=4.4.0,<=4.12.0 # Apache Software License oslo.config>=4.6.0 # Apache Software License
oslo.db>=4.24.0,<=4.26.0 # Apache Software License oslo.db>=4.27.0 # Apache Software License
oslo.log==3.30.0 # Apache Software License oslo.log>=3.30.0 # Apache Software License
oslo.utils>=3.20.0,<=3.29.0 # Apache Software License oslo.utils>=3.28.0 # Apache Software License
paramiko>=2.0.0,<=2.2.1 # LGPL paramiko>=2.0.0 # LGPL
pbr>=2.0.0,!=2.1.0,<=3.1.1 # Apache Software License pbr>=2.0.0,!=2.1.0 # Apache Software License
PrettyTable>=0.7.1,<0.8 # BSD PrettyTable>=0.7.1,<0.8 # BSD
PyYAML>=3.10,<=3.12 # MIT PyYAML>=3.10 # MIT
python-subunit>=0.0.18,<=1.2.0 python-subunit>=0.0.18 # UNKNOWN
requests>=2.14.2,<=2.18.4 # Apache License, Version 2.0 requests>=2.14.2 # Apache License, Version 2.0
SQLAlchemy>=1.0.10,!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,<=1.2.0b2 # MIT SQLAlchemy>=1.0.10,!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT
six>=1.9.0,<=1.10.0 # MIT six>=1.9.0 # MIT
virtualenv>=13.1.0,<=15.1.0 # MIT virtualenv>=13.1.0 # MIT
# OpenStack related # OpenStack related
boto>=2.32.1,<=2.48.0 # MIT boto>=2.32.1 # MIT
gnocchiclient>=2.7.0,<=4.0.0 # Apache Software License gnocchiclient>=3.3.1 # Apache Software License
keystoneauth1==3.2.0 # Apache Software License keystoneauth1>=3.2.0 # Apache Software License
os-faults>=0.1.15,<0.2.0 # Apache Software License os-faults>=0.1.15 # Apache Software License
osprofiler>=1.4.0,<=1.12.0 # Apache Software License osprofiler>=1.4.0 # Apache Software License
python-ceilometerclient>=2.5.0,<=2.9.0 # Apache Software License python-ceilometerclient>=2.5.0 # Apache Software License
python-cinderclient==3.2.0 # Apache Software License python-cinderclient>=3.2.0 # Apache Software License
python-designateclient>=1.5.0,<=2.7.0 # Apache License, Version 2.0 python-designateclient>=2.7.0 # Apache License, Version 2.0
python-heatclient>=1.6.1,<=1.11.0 # Apache Software License python-heatclient>=1.10.0 # Apache Software License
python-glanceclient==2.8.0 # Apache License, Version 2.0 python-glanceclient>=2.8.0 # Apache License, Version 2.0
python-ironicclient>=1.14.0,<=1.17.0 # Apache Software License python-ironicclient>=1.14.0 # Apache Software License
python-keystoneclient>=3.8.0,<=3.13.0 # Apache Software License python-keystoneclient>=3.8.0 # Apache Software License
python-magnumclient>=2.0.0,<=2.7.0 # Apache Software License python-magnumclient>=2.0.0 # Apache Software License
python-manilaclient>=1.12.0,<=1.17.1 # Apache Software License python-manilaclient>=1.16.0 # Apache Software License
python-mistralclient>=3.1.0,<=3.1.2 # Apache Software License python-mistralclient>=3.1.0 # Apache Software License
python-muranoclient>=0.8.2,<=0.14.0 # Apache License, Version 2.0 python-muranoclient>=0.8.2 # Apache License, Version 2.0
python-monascaclient==1.7.0 # Apache Software License python-monascaclient>=1.7.0 # Apache Software License
python-neutronclient>=6.3.0,<=6.5.0 # Apache Software License python-neutronclient>=6.3.0 # Apache Software License
python-novaclient==9.1.0 # Apache License, Version 2.0 python-novaclient>=9.1.0 # Apache License, Version 2.0
python-saharaclient>=1.1.0,<=1.3.0 # Apache License, Version 2.0 python-saharaclient>=1.2.0 # Apache License, Version 2.0
python-senlinclient>=1.1.0,<=1.4.0 # Apache Software License python-senlinclient>=1.1.0 # Apache Software License
python-swiftclient>=3.2.0,<=3.4.0 # Apache Software License python-swiftclient>=3.2.0 # Apache Software License
python-troveclient>=2.2.0,<=2.12.0 # Apache Software License python-troveclient>=2.2.0 # Apache Software License
python-watcherclient>=0.23.0,<=1.3.0 # Apache Software License python-watcherclient>=1.1.0 # Apache Software License
python-zaqarclient>=1.0.0,<=1.7.0 # Apache Software License python-zaqarclient>=1.0.0 # Apache Software License
kubernetes>=1.0.0,<=3.0.0 # Apache License Version 2.0 kubernetes>=1.0.0 # Apache License Version 2.0

View File

@ -2,25 +2,25 @@
# of appearance. Changing the order has an impact on the overall integration # of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later. # process, which may cause wedges in the gate later.
hacking>=0.12.0,!=0.13.0,<=1.0.0 # Apache Software License hacking>=0.12.0,!=0.13.0 # Apache Software License
pytest>=2.7,<=3.2.1 # MIT pytest>=2.7 # MIT
# py.test plugin for measuring coverage. # py.test plugin for measuring coverage.
pytest-cov>=2.2.1,<=2.5.1 # MIT pytest-cov>=2.2.1 # MIT
# py.test plugin for generating HTML reports # py.test plugin for generating HTML reports
pytest-html>=1.10.0,<=1.15.2 # Mozilla Public License 2.0 (MPL 2.0) pytest-html>=1.10.0 # Mozilla Public License 2.0 (MPL 2.0)
# py.test xdist plugin for distributed testing and loop-on-failing modes # py.test xdist plugin for distributed testing and loop-on-failing modes
pytest-xdist<=1.20.0 # MIT pytest-xdist # MIT
coverage>=4.0,!=4.4,<=4.4.1 # Apache License, Version 2.0 coverage>=4.0,!=4.4 # Apache License, Version 2.0
ddt>=1.0.1,<=1.1.1 ddt>=1.0.1 # UNKNOWN
mock==2.0.0 mock>=2.0.0 # UNKNOWN
python-dateutil>=2.4.2,<=2.6.1 # Simplified BSD python-dateutil>=2.4.2 # Simplified BSD
testtools>=1.4.0,<=2.3.0 testtools>=1.4.0 # UNKNOWN
sphinx>=1.6.2,<=1.6.3 # BSD sphinx>=1.6.2 # BSD
oslosphinx>=4.7.0,<=4.15.1 # Apache Software License oslosphinx>=4.7.0 # Apache Software License
oslotest>=1.10.0,<=2.17.0 # Apache Software License oslotest>=1.10.0 # Apache Software License
testresources>=0.2.4,<=2.0.1 testresources>=2.0.0 # UNKNOWN
testscenarios>=0.4,<=0.5.0 testscenarios>=0.4 # UNKNOWN

View File

@ -18,7 +18,7 @@ Synchronizes, formats and prepares requirements to release(obtains and adds
maximum allowed version). maximum allowed version).
""" """
import argparse import collections
import logging import logging
import re import re
import sys import sys
@ -37,7 +37,6 @@ GLOBAL_REQUIREMENTS_LOCATIONS = (
"https://raw.githubusercontent.com/openstack/requirements/master/", "https://raw.githubusercontent.com/openstack/requirements/master/",
"http://git.openstack.org/cgit/openstack/requirements/plain/" "http://git.openstack.org/cgit/openstack/requirements/plain/"
) )
GLOBAL_REQUIREMENTS_FILENAME = "global-requirements.txt"
RALLY_REQUIREMENTS_FILES = ( RALLY_REQUIREMENTS_FILES = (
"requirements.txt", "requirements.txt",
"test-requirements.txt" "test-requirements.txt"
@ -63,12 +62,10 @@ class Comment(object):
initial_indent="# ", subsequent_indent="# ") initial_indent="# ", subsequent_indent="# ")
class Requirement(object): _PYPI_CACHE = {}
RE_NAME = re.compile(r"[a-zA-Z0-9-._]+")
RE_CONST_VERSION = re.compile(r"==[a-zA-Z0-9.]+")
RE_MIN_VERSION = re.compile(r">=?[a-zA-Z0-9.]+") class PYPIPackage(object):
RE_MAX_VERSION = re.compile(r"<=?[a-zA-Z0-9.]+")
RE_NE_VERSIONS = re.compile(r"!=[a-zA-Z0-9.]+")
# NOTE(andreykurilin): one license can have different labels. Let's use # NOTE(andreykurilin): one license can have different labels. Let's use
# unified variant. # unified variant.
LICENSE_MAP = {"MIT license": "MIT", LICENSE_MAP = {"MIT license": "MIT",
@ -76,44 +73,73 @@ class Requirement(object):
"BSD License": "BSD", "BSD License": "BSD",
"Apache 2.0": "Apache License, Version 2.0"} "Apache 2.0": "Apache License, Version 2.0"}
def __init__(self, package_name, version): def __init__(self, package_name):
self.package_name = package_name self.package_name = package_name
self.version = version self._pypi_info = None
self._license = None self._pypi_license = None
self._pypy_info = None
self.do_not_touch = False
def sync_max_version_with_pypy(self):
if isinstance(self.version, dict) and not self.do_not_touch:
self.version["max"] = "<=%s" % self.pypy_info["info"]["version"]
@property @property
def pypy_info(self): def pypi_info(self):
if self._pypy_info is None: if self._pypi_info is None:
if self.package_name in _PYPI_CACHE:
self._pypi_info = _PYPI_CACHE[self.package_name]
else:
resp = requests.get("https://pypi.python.org/pypi/%s/json" % resp = requests.get("https://pypi.python.org/pypi/%s/json" %
self.package_name) self.package_name)
if resp.status_code != 200: if resp.status_code != 200:
print("An error occurred while checking '%s' package at "
"pypi." % self.package_name)
raise Exception(resp.text) raise Exception(resp.text)
self._pypy_info = resp.json() self._pypi_info = resp.json()
return self._pypy_info
# let's cache it for the case when we need to sync requirements
# and update upper constrains
_PYPI_CACHE[self.package_name] = self._pypi_info
return self._pypi_info
@property @property
def license(self): def pypi_version(self):
if self._license is None: return self.pypi_info["info"]["version"]
if self.pypy_info["info"]["license"]:
self._license = self.pypy_info["info"]["license"] @property
def pypi_license(self):
if self._pypi_license is None:
if self.pypi_info["info"]["license"]:
self._pypi_license = self.pypi_info["info"]["license"]
else: else:
# try to parse classifiers # try to parse classifiers
prefix = "License :: OSI Approved :: " prefix = "License :: OSI Approved :: "
classifiers = [c[len(prefix):] classifiers = [c[len(prefix):]
for c in self.pypy_info["info"]["classifiers"] for c in self.pypi_info["info"]["classifiers"]
if c.startswith(prefix)] if c.startswith(prefix)]
self._license = "/".join(classifiers) self._pypi_license = "/".join(classifiers)
self._license = self.LICENSE_MAP.get(self._license, self._license) self._license = self.LICENSE_MAP.get(self._pypi_license,
if self._license == "UNKNOWN": self._pypi_license)
self._license = None if self._pypi_license == "UNKNOWN":
self._pypi_license = None
return self._license return self._license
def __eq__(self, other):
return (isinstance(other, PYPIPackage) and
self.package_name == other.package_name)
class Requirement(PYPIPackage):
RE_NAME = re.compile(r"[a-zA-Z0-9-._]+")
RE_CONST_VERSION = re.compile(r"==[a-zA-Z0-9.]+")
RE_MIN_VERSION = re.compile(r">=?[a-zA-Z0-9.]+")
RE_MAX_VERSION = re.compile(r"<=?[a-zA-Z0-9.]+")
RE_NE_VERSIONS = re.compile(r"!=[a-zA-Z0-9.]+")
def __init__(self, package_name, version):
super(Requirement, self).__init__(package_name)
self.version = version
self.do_not_touch = False
def sync_max_version_with_pypy(self):
if isinstance(self.version, dict) and not self.do_not_touch:
self.version["max"] = "<=%s" % self.pypi_version
@classmethod @classmethod
def parse_line(cls, line): def parse_line(cls, line):
match = cls.RE_NAME.match(line) match = cls.RE_NAME.match(line)
@ -180,7 +206,7 @@ class Requirement(object):
version = ">=%s" % self.version[2:] version = ">=%s" % self.version[2:]
string = "%s%s" % (self.package_name, version) string = "%s%s" % (self.package_name, version)
if self.license: if self.pypi_license:
# NOTE(andreykurilin): When I start implementation of this script, # NOTE(andreykurilin): When I start implementation of this script,
# python-keystoneclient dependency string took around ~45-55 # python-keystoneclient dependency string took around ~45-55
# chars, so let's use this length as indent. Feel free to modify # chars, so let's use this length as indent. Feel free to modify
@ -190,7 +216,7 @@ class Requirement(object):
indent = magic_number - len(string) indent = magic_number - len(string)
else: else:
indent = 2 indent = 2
string += " " * indent + "# " + self.license string += " " * indent + "# " + self.pypi_license
return string return string
def __eq__(self, other): def __eq__(self, other):
@ -201,7 +227,35 @@ class Requirement(object):
return not self.__eq__(other) return not self.__eq__(other)
def parse_data(raw_data, include_comments=True): class UpperConstraint(PYPIPackage):
RE_LINE = re.compile(
r"(?P<package_name>[a-zA-Z0-9-._]+)===(?P<version>[a-zA-Z0-9.]+)")
def __init__(self, package_name, version=None):
super(UpperConstraint, self).__init__(package_name)
self._version = version
def __str__(self):
return "%s===%s" % (self.package_name, self.version)
@property
def version(self):
if self._version is None:
self._version = self.pypi_version
return self._version
@classmethod
def parse_line(cls, line):
match = cls.RE_LINE.match(line)
if match:
return cls(**match.groupdict())
def update(self, version):
self._version = version
def parse_data(raw_data, include_comments=True, dependency_cls=Requirement):
# first elem is None to simplify checks of last elem in requirements # first elem is None to simplify checks of last elem in requirements
requirements = [None] requirements = [None]
for line in raw_data.split("\n"): for line in raw_data.split("\n"):
@ -222,13 +276,15 @@ def parse_data(raw_data, include_comments=True):
if (isinstance(requirements[-1], Comment) and if (isinstance(requirements[-1], Comment) and
not requirements[-1].is_finished): not requirements[-1].is_finished):
requirements[-1].finish_him() requirements[-1].finish_him()
# parse_line # parse_line
req = Requirement.parse_line(line) dep = dependency_cls.parse_line(line)
if req: if dep:
if (isinstance(requirements[-1], Comment) and if (isinstance(requirements[-1], Comment) and
DO_NOT_TOUCH_TAG in str(requirements[-1])): DO_NOT_TOUCH_TAG in str(requirements[-1])):
req.do_not_touch = True dep.do_not_touch = True
requirements.append(req) requirements.append(dep)
for i in range(len(requirements) - 1, 0, -1): for i in range(len(requirements) - 1, 0, -1):
# remove empty lines at the end of file # remove empty lines at the end of file
if isinstance(requirements[i], Comment): if isinstance(requirements[i], Comment):
@ -236,22 +292,27 @@ def parse_data(raw_data, include_comments=True):
requirements.pop(i) requirements.pop(i)
else: else:
break break
return requirements[1:] return collections.OrderedDict(
(v if isinstance(v, Comment) else v.package_name, v)
for v in requirements if v)
def _read_requirements(): def _fetch_from_gr(filename):
"""Read all rally requirements.""" """Try to fetch data from OpenStack global-requirements repo"""
LOG.info("Reading rally requirements...") for i in range(0, len(GLOBAL_REQUIREMENTS_LOCATIONS)):
for file_name in RALLY_REQUIREMENTS_FILES: url = GLOBAL_REQUIREMENTS_LOCATIONS[i] + filename
LOG.debug("Try to read '%s'." % file_name) LOG.debug("Try to obtain %s from %s" % (filename, url))
with open(file_name) as f: try:
data = f.read() return requests.get(url).text
LOG.info("Parsing requirements from %s." % file_name) except requests.ConnectionError as e:
yield file_name, parse_data(data) LOG.exception(e)
raise Exception("Unable to obtain %s" % filename)
def _write_requirements(filename, requirements): def _write_requirements(filename, requirements):
"""Saves requirements to file.""" """Saves requirements to file."""
if isinstance(requirements, dict):
requirements = requirements.values()
LOG.info("Saving requirements to %s." % filename) LOG.info("Saving requirements to %s." % filename)
with open(filename, "w") as f: with open(filename, "w") as f:
for entity in requirements: for entity in requirements:
@ -259,82 +320,66 @@ def _write_requirements(filename, requirements):
f.write("\n") f.write("\n")
def _sync(): def sync_requirements():
LOG.info("Obtaining global-requirements...")
for i in range(0, len(GLOBAL_REQUIREMENTS_LOCATIONS)):
url = GLOBAL_REQUIREMENTS_LOCATIONS[i] + GLOBAL_REQUIREMENTS_FILENAME
LOG.debug("Try to obtain global-requirements from %s" % url)
try:
raw_gr = requests.get(url).text
except requests.ConnectionError as e:
LOG.exception(e)
if i == len(GLOBAL_REQUIREMENTS_LOCATIONS) - 1:
# there are no more urls to try
raise Exception("Unable to obtain %s" %
GLOBAL_REQUIREMENTS_FILENAME)
else:
break
LOG.info("Parsing global-requirements...")
# NOTE(andreykurilin): global-requirements includes comments which can be
# unrelated to Rally project.
gr = parse_data(raw_gr, include_comments=False)
for filename, requirements in _read_requirements():
for i in range(0, len(requirements)):
if (isinstance(requirements[i], Requirement) and
not requirements[i].do_not_touch):
try:
gr_item = gr[gr.index(requirements[i])]
except ValueError:
# it not g-r requirements
if isinstance(requirements[i].version, dict):
requirements[i].version["max"] = None
else:
requirements[i].version = gr_item.version
yield filename, requirements
def sync():
"""Synchronizes Rally requirements with OpenStack global-requirements.""" """Synchronizes Rally requirements with OpenStack global-requirements."""
for filename, requirements in _sync(): LOG.info("Obtaining global-requirements of OpenStack...")
_write_requirements(filename, requirements) raw_gr = _fetch_from_gr("global-requirements.txt")
# NOTE(andreykurilin): global-requirements includes comments which can be
# unrelated to Rally project, so let's just ignore them
gr = parse_data(raw_gr, include_comments=False)
for file_name in RALLY_REQUIREMENTS_FILES:
LOG.debug("Processing '%s'." % file_name)
with open(file_name) as f:
requirements = parse_data(f.read())
for name, req in requirements.items():
if isinstance(req, Requirement) and not req.do_not_touch:
if name in gr:
req.version = gr[req.package_name].version
else:
# it not g-r requirements
if isinstance(req.version, dict):
req.version["max"] = None
_write_requirements(file_name, requirements)
def format_requirements(): def update_upper_constraints():
"""Obtain package licenses from pypy and write requirements to file.""" """Obtains latest version of packages and put them to upper-constraints."""
for filename, requirements in _read_requirements(): LOG.info("Obtaining upper-constrains from OpenStack...")
_write_requirements(filename, requirements) raw_g_uc = _fetch_from_gr("upper-constraints.txt")
# NOTE(andreykurilin): global OpenStack upper-constraints file includes
# comments which can be unrelated to Rally project, so let's just ignore
# them.
global_uc = parse_data(raw_g_uc, include_comments=False,
dependency_cls=UpperConstraint)
with open("upper-constraints.txt") as f:
our_uc = parse_data(f.read(), dependency_cls=UpperConstraint)
with open("requirements.txt") as f:
our_requirements = parse_data(f.read(), include_comments=False)
for name, req in our_requirements.items():
if isinstance(req, Comment):
print("continue")
continue
print(req)
if name not in our_uc:
our_uc[name] = UpperConstraint(name)
def add_uppers(): if name in global_uc:
"""Obtains latest version of packages and put them to requirements.""" # we cannot use whatever we want versions in CI. OpenStack CI
for filename, requirements in _sync(): # installs ignores versions listed in requirements of
LOG.info("Obtaining latest versions of packages for %s." % filename) # particular project and use versions from global u-c file.
for req in requirements: # It means that we need to suggest to use the same versions
if isinstance(req, Requirement): our_uc[name].update(global_uc[name].version)
if isinstance(req.version, dict) and not req.version["max"]:
req.sync_max_version_with_pypy() our_uc = sorted(our_uc.values(), key=lambda o: o.package_name.upper())
_write_requirements(filename, requirements) _write_requirements("upper-constraints.txt", our_uc)
def main(): def main():
parser = argparse.ArgumentParser( sync_requirements()
prog="Python Requirement Manager for Rally", update_upper_constraints()
description=__doc__.strip(),
add_help=True
)
action_groups = parser.add_mutually_exclusive_group()
action_groups.add_argument("--format",
action="store_const",
const=format_requirements,
dest="action")
action_groups.add_argument("--add-upper",
action="store_const",
const=add_uppers,
dest="action")
action_groups.set_defaults(action=sync)
parser.parse_args(sys.argv[1:]).action()
if __name__ == "__main__": if __name__ == "__main__":
sys.exit(main()) sys.exit(main())

45
upper-constraints.txt Normal file
View File

@ -0,0 +1,45 @@
alembic===0.9.6
boto===2.48.0
decorator===4.1.2
gnocchiclient===4.0.0
Jinja2===2.9.6
jsonschema===2.6.0
keystoneauth1===3.2.0
kubernetes===2.0.0
morph===0.1.2
netaddr===0.7.19
os-faults===0.1.17
oslo.config===5.0.0
oslo.db===4.29.0
oslo.log===3.32.0
oslo.utils===3.30.0
osprofiler===1.13.0
paramiko===2.3.1
pbr===3.1.1
PrettyTable===7
python-ceilometerclient===2.9.0
python-cinderclient===3.2.0
python-designateclient===2.7.0
python-glanceclient===2.8.0
python-heatclient===1.12.0
python-ironicclient===1.17.1
python-keystoneclient===3.13.0
python-magnumclient===2.7.0
python-manilaclient===1.17.2
python-mistralclient===3.1.3
python-monascaclient===1.8.0
python-muranoclient===0.14.0
python-neutronclient===6.5.0
python-novaclient===9.1.1
python-saharaclient===1.3.0
python-senlinclient===1.4.0
python-subunit===1.2.0
python-swiftclient===3.4.0
python-troveclient===2.12.0
python-watcherclient===1.4.0
python-zaqarclient===1.7.0
PyYAML===3.12
requests===2.18.4
six===1.11.0
SQLAlchemy===1.1.14
virtualenv===15.1.0