[requirements] Stop using upper limits for all cases
Using upper limits for all requirements adds a lot of pain for operators and packaging teams. Sometimes it is very hard to find a list of compatible packages which will feet rally and some other libs. This patch stops setting upper limits for all packages in our requirements list and moves to our personal upper-constraints file. PS: It doesn't mean that we cannot limit packages in case of failures and known compatibility issues Change-Id: Id5d84fd1b605811ecbf7f88a4af8c2c607b9dd72
This commit is contained in:
parent
7153e0cbc5
commit
d39eed2be7
@ -3,50 +3,50 @@
|
|||||||
# process, which may cause wedges in the gate later.
|
# process, which may cause wedges in the gate later.
|
||||||
|
|
||||||
# Rally core dependencies
|
# Rally core dependencies
|
||||||
alembic>=0.8.10,<=0.9.5 # MIT
|
alembic>=0.8.10 # MIT
|
||||||
decorator>=3.4.0,<=4.1.2 # new BSD License
|
decorator>=3.4.0 # new BSD License
|
||||||
Jinja2>=2.8,!=2.9.0,!=2.9.1,!=2.9.2,!=2.9.3,!=2.9.4,<=2.9.6 # BSD
|
Jinja2>=2.8,!=2.9.0,!=2.9.1,!=2.9.2,!=2.9.3,!=2.9.4 # BSD
|
||||||
jsonschema>=2.0.0,!=2.5.0,<3.0.0 # MIT
|
jsonschema>=2.6.0,<3.0.0 # MIT
|
||||||
morph
|
morph # GPLv3+
|
||||||
netaddr>=0.7.13,!=0.7.16,<=0.7.19 # BSD
|
netaddr>=0.7.18 # BSD
|
||||||
oslo.config>=4.0.0,!=4.3.0,!=4.4.0,<=4.12.0 # Apache Software License
|
oslo.config>=4.6.0 # Apache Software License
|
||||||
oslo.db>=4.24.0,<=4.26.0 # Apache Software License
|
oslo.db>=4.27.0 # Apache Software License
|
||||||
oslo.log==3.30.0 # Apache Software License
|
oslo.log>=3.30.0 # Apache Software License
|
||||||
oslo.utils>=3.20.0,<=3.29.0 # Apache Software License
|
oslo.utils>=3.28.0 # Apache Software License
|
||||||
paramiko>=2.0.0,<=2.2.1 # LGPL
|
paramiko>=2.0.0 # LGPL
|
||||||
pbr>=2.0.0,!=2.1.0,<=3.1.1 # Apache Software License
|
pbr>=2.0.0,!=2.1.0 # Apache Software License
|
||||||
PrettyTable>=0.7.1,<0.8 # BSD
|
PrettyTable>=0.7.1,<0.8 # BSD
|
||||||
PyYAML>=3.10,<=3.12 # MIT
|
PyYAML>=3.10 # MIT
|
||||||
python-subunit>=0.0.18,<=1.2.0
|
python-subunit>=0.0.18 # UNKNOWN
|
||||||
requests>=2.14.2,<=2.18.4 # Apache License, Version 2.0
|
requests>=2.14.2 # Apache License, Version 2.0
|
||||||
SQLAlchemy>=1.0.10,!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,<=1.2.0b2 # MIT
|
SQLAlchemy>=1.0.10,!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT
|
||||||
six>=1.9.0,<=1.10.0 # MIT
|
six>=1.9.0 # MIT
|
||||||
virtualenv>=13.1.0,<=15.1.0 # MIT
|
virtualenv>=13.1.0 # MIT
|
||||||
|
|
||||||
# OpenStack related
|
# OpenStack related
|
||||||
boto>=2.32.1,<=2.48.0 # MIT
|
boto>=2.32.1 # MIT
|
||||||
gnocchiclient>=2.7.0,<=4.0.0 # Apache Software License
|
gnocchiclient>=3.3.1 # Apache Software License
|
||||||
keystoneauth1==3.2.0 # Apache Software License
|
keystoneauth1>=3.2.0 # Apache Software License
|
||||||
os-faults>=0.1.15,<0.2.0 # Apache Software License
|
os-faults>=0.1.15 # Apache Software License
|
||||||
osprofiler>=1.4.0,<=1.12.0 # Apache Software License
|
osprofiler>=1.4.0 # Apache Software License
|
||||||
python-ceilometerclient>=2.5.0,<=2.9.0 # Apache Software License
|
python-ceilometerclient>=2.5.0 # Apache Software License
|
||||||
python-cinderclient==3.2.0 # Apache Software License
|
python-cinderclient>=3.2.0 # Apache Software License
|
||||||
python-designateclient>=1.5.0,<=2.7.0 # Apache License, Version 2.0
|
python-designateclient>=2.7.0 # Apache License, Version 2.0
|
||||||
python-heatclient>=1.6.1,<=1.11.0 # Apache Software License
|
python-heatclient>=1.10.0 # Apache Software License
|
||||||
python-glanceclient==2.8.0 # Apache License, Version 2.0
|
python-glanceclient>=2.8.0 # Apache License, Version 2.0
|
||||||
python-ironicclient>=1.14.0,<=1.17.0 # Apache Software License
|
python-ironicclient>=1.14.0 # Apache Software License
|
||||||
python-keystoneclient>=3.8.0,<=3.13.0 # Apache Software License
|
python-keystoneclient>=3.8.0 # Apache Software License
|
||||||
python-magnumclient>=2.0.0,<=2.7.0 # Apache Software License
|
python-magnumclient>=2.0.0 # Apache Software License
|
||||||
python-manilaclient>=1.12.0,<=1.17.1 # Apache Software License
|
python-manilaclient>=1.16.0 # Apache Software License
|
||||||
python-mistralclient>=3.1.0,<=3.1.2 # Apache Software License
|
python-mistralclient>=3.1.0 # Apache Software License
|
||||||
python-muranoclient>=0.8.2,<=0.14.0 # Apache License, Version 2.0
|
python-muranoclient>=0.8.2 # Apache License, Version 2.0
|
||||||
python-monascaclient==1.7.0 # Apache Software License
|
python-monascaclient>=1.7.0 # Apache Software License
|
||||||
python-neutronclient>=6.3.0,<=6.5.0 # Apache Software License
|
python-neutronclient>=6.3.0 # Apache Software License
|
||||||
python-novaclient==9.1.0 # Apache License, Version 2.0
|
python-novaclient>=9.1.0 # Apache License, Version 2.0
|
||||||
python-saharaclient>=1.1.0,<=1.3.0 # Apache License, Version 2.0
|
python-saharaclient>=1.2.0 # Apache License, Version 2.0
|
||||||
python-senlinclient>=1.1.0,<=1.4.0 # Apache Software License
|
python-senlinclient>=1.1.0 # Apache Software License
|
||||||
python-swiftclient>=3.2.0,<=3.4.0 # Apache Software License
|
python-swiftclient>=3.2.0 # Apache Software License
|
||||||
python-troveclient>=2.2.0,<=2.12.0 # Apache Software License
|
python-troveclient>=2.2.0 # Apache Software License
|
||||||
python-watcherclient>=0.23.0,<=1.3.0 # Apache Software License
|
python-watcherclient>=1.1.0 # Apache Software License
|
||||||
python-zaqarclient>=1.0.0,<=1.7.0 # Apache Software License
|
python-zaqarclient>=1.0.0 # Apache Software License
|
||||||
kubernetes>=1.0.0,<=3.0.0 # Apache License Version 2.0
|
kubernetes>=1.0.0 # Apache License Version 2.0
|
||||||
|
@ -2,25 +2,25 @@
|
|||||||
# of appearance. Changing the order has an impact on the overall integration
|
# of appearance. Changing the order has an impact on the overall integration
|
||||||
# process, which may cause wedges in the gate later.
|
# process, which may cause wedges in the gate later.
|
||||||
|
|
||||||
hacking>=0.12.0,!=0.13.0,<=1.0.0 # Apache Software License
|
hacking>=0.12.0,!=0.13.0 # Apache Software License
|
||||||
|
|
||||||
pytest>=2.7,<=3.2.1 # MIT
|
pytest>=2.7 # MIT
|
||||||
# py.test plugin for measuring coverage.
|
# py.test plugin for measuring coverage.
|
||||||
pytest-cov>=2.2.1,<=2.5.1 # MIT
|
pytest-cov>=2.2.1 # MIT
|
||||||
# py.test plugin for generating HTML reports
|
# py.test plugin for generating HTML reports
|
||||||
pytest-html>=1.10.0,<=1.15.2 # Mozilla Public License 2.0 (MPL 2.0)
|
pytest-html>=1.10.0 # Mozilla Public License 2.0 (MPL 2.0)
|
||||||
# py.test xdist plugin for distributed testing and loop-on-failing modes
|
# py.test xdist plugin for distributed testing and loop-on-failing modes
|
||||||
pytest-xdist<=1.20.0 # MIT
|
pytest-xdist # MIT
|
||||||
|
|
||||||
coverage>=4.0,!=4.4,<=4.4.1 # Apache License, Version 2.0
|
coverage>=4.0,!=4.4 # Apache License, Version 2.0
|
||||||
ddt>=1.0.1,<=1.1.1
|
ddt>=1.0.1 # UNKNOWN
|
||||||
mock==2.0.0
|
mock>=2.0.0 # UNKNOWN
|
||||||
python-dateutil>=2.4.2,<=2.6.1 # Simplified BSD
|
python-dateutil>=2.4.2 # Simplified BSD
|
||||||
testtools>=1.4.0,<=2.3.0
|
testtools>=1.4.0 # UNKNOWN
|
||||||
|
|
||||||
sphinx>=1.6.2,<=1.6.3 # BSD
|
sphinx>=1.6.2 # BSD
|
||||||
oslosphinx>=4.7.0,<=4.15.1 # Apache Software License
|
oslosphinx>=4.7.0 # Apache Software License
|
||||||
oslotest>=1.10.0,<=2.17.0 # Apache Software License
|
oslotest>=1.10.0 # Apache Software License
|
||||||
|
|
||||||
testresources>=0.2.4,<=2.0.1
|
testresources>=2.0.0 # UNKNOWN
|
||||||
testscenarios>=0.4,<=0.5.0
|
testscenarios>=0.4 # UNKNOWN
|
||||||
|
@ -18,7 +18,7 @@ Synchronizes, formats and prepares requirements to release(obtains and adds
|
|||||||
maximum allowed version).
|
maximum allowed version).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import collections
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
@ -37,7 +37,6 @@ GLOBAL_REQUIREMENTS_LOCATIONS = (
|
|||||||
"https://raw.githubusercontent.com/openstack/requirements/master/",
|
"https://raw.githubusercontent.com/openstack/requirements/master/",
|
||||||
"http://git.openstack.org/cgit/openstack/requirements/plain/"
|
"http://git.openstack.org/cgit/openstack/requirements/plain/"
|
||||||
)
|
)
|
||||||
GLOBAL_REQUIREMENTS_FILENAME = "global-requirements.txt"
|
|
||||||
RALLY_REQUIREMENTS_FILES = (
|
RALLY_REQUIREMENTS_FILES = (
|
||||||
"requirements.txt",
|
"requirements.txt",
|
||||||
"test-requirements.txt"
|
"test-requirements.txt"
|
||||||
@ -63,12 +62,10 @@ class Comment(object):
|
|||||||
initial_indent="# ", subsequent_indent="# ")
|
initial_indent="# ", subsequent_indent="# ")
|
||||||
|
|
||||||
|
|
||||||
class Requirement(object):
|
_PYPI_CACHE = {}
|
||||||
RE_NAME = re.compile(r"[a-zA-Z0-9-._]+")
|
|
||||||
RE_CONST_VERSION = re.compile(r"==[a-zA-Z0-9.]+")
|
|
||||||
RE_MIN_VERSION = re.compile(r">=?[a-zA-Z0-9.]+")
|
class PYPIPackage(object):
|
||||||
RE_MAX_VERSION = re.compile(r"<=?[a-zA-Z0-9.]+")
|
|
||||||
RE_NE_VERSIONS = re.compile(r"!=[a-zA-Z0-9.]+")
|
|
||||||
# NOTE(andreykurilin): one license can have different labels. Let's use
|
# NOTE(andreykurilin): one license can have different labels. Let's use
|
||||||
# unified variant.
|
# unified variant.
|
||||||
LICENSE_MAP = {"MIT license": "MIT",
|
LICENSE_MAP = {"MIT license": "MIT",
|
||||||
@ -76,44 +73,73 @@ class Requirement(object):
|
|||||||
"BSD License": "BSD",
|
"BSD License": "BSD",
|
||||||
"Apache 2.0": "Apache License, Version 2.0"}
|
"Apache 2.0": "Apache License, Version 2.0"}
|
||||||
|
|
||||||
def __init__(self, package_name, version):
|
def __init__(self, package_name):
|
||||||
self.package_name = package_name
|
self.package_name = package_name
|
||||||
self.version = version
|
self._pypi_info = None
|
||||||
self._license = None
|
self._pypi_license = None
|
||||||
self._pypy_info = None
|
|
||||||
self.do_not_touch = False
|
|
||||||
|
|
||||||
def sync_max_version_with_pypy(self):
|
|
||||||
if isinstance(self.version, dict) and not self.do_not_touch:
|
|
||||||
self.version["max"] = "<=%s" % self.pypy_info["info"]["version"]
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def pypy_info(self):
|
def pypi_info(self):
|
||||||
if self._pypy_info is None:
|
if self._pypi_info is None:
|
||||||
resp = requests.get("https://pypi.python.org/pypi/%s/json" %
|
if self.package_name in _PYPI_CACHE:
|
||||||
self.package_name)
|
self._pypi_info = _PYPI_CACHE[self.package_name]
|
||||||
if resp.status_code != 200:
|
else:
|
||||||
raise Exception(resp.text)
|
resp = requests.get("https://pypi.python.org/pypi/%s/json" %
|
||||||
self._pypy_info = resp.json()
|
self.package_name)
|
||||||
return self._pypy_info
|
if resp.status_code != 200:
|
||||||
|
print("An error occurred while checking '%s' package at "
|
||||||
|
"pypi." % self.package_name)
|
||||||
|
raise Exception(resp.text)
|
||||||
|
self._pypi_info = resp.json()
|
||||||
|
|
||||||
|
# let's cache it for the case when we need to sync requirements
|
||||||
|
# and update upper constrains
|
||||||
|
_PYPI_CACHE[self.package_name] = self._pypi_info
|
||||||
|
return self._pypi_info
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def license(self):
|
def pypi_version(self):
|
||||||
if self._license is None:
|
return self.pypi_info["info"]["version"]
|
||||||
if self.pypy_info["info"]["license"]:
|
|
||||||
self._license = self.pypy_info["info"]["license"]
|
@property
|
||||||
|
def pypi_license(self):
|
||||||
|
if self._pypi_license is None:
|
||||||
|
if self.pypi_info["info"]["license"]:
|
||||||
|
self._pypi_license = self.pypi_info["info"]["license"]
|
||||||
else:
|
else:
|
||||||
# try to parse classifiers
|
# try to parse classifiers
|
||||||
prefix = "License :: OSI Approved :: "
|
prefix = "License :: OSI Approved :: "
|
||||||
classifiers = [c[len(prefix):]
|
classifiers = [c[len(prefix):]
|
||||||
for c in self.pypy_info["info"]["classifiers"]
|
for c in self.pypi_info["info"]["classifiers"]
|
||||||
if c.startswith(prefix)]
|
if c.startswith(prefix)]
|
||||||
self._license = "/".join(classifiers)
|
self._pypi_license = "/".join(classifiers)
|
||||||
self._license = self.LICENSE_MAP.get(self._license, self._license)
|
self._license = self.LICENSE_MAP.get(self._pypi_license,
|
||||||
if self._license == "UNKNOWN":
|
self._pypi_license)
|
||||||
self._license = None
|
if self._pypi_license == "UNKNOWN":
|
||||||
|
self._pypi_license = None
|
||||||
return self._license
|
return self._license
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
return (isinstance(other, PYPIPackage) and
|
||||||
|
self.package_name == other.package_name)
|
||||||
|
|
||||||
|
|
||||||
|
class Requirement(PYPIPackage):
|
||||||
|
RE_NAME = re.compile(r"[a-zA-Z0-9-._]+")
|
||||||
|
RE_CONST_VERSION = re.compile(r"==[a-zA-Z0-9.]+")
|
||||||
|
RE_MIN_VERSION = re.compile(r">=?[a-zA-Z0-9.]+")
|
||||||
|
RE_MAX_VERSION = re.compile(r"<=?[a-zA-Z0-9.]+")
|
||||||
|
RE_NE_VERSIONS = re.compile(r"!=[a-zA-Z0-9.]+")
|
||||||
|
|
||||||
|
def __init__(self, package_name, version):
|
||||||
|
super(Requirement, self).__init__(package_name)
|
||||||
|
self.version = version
|
||||||
|
self.do_not_touch = False
|
||||||
|
|
||||||
|
def sync_max_version_with_pypy(self):
|
||||||
|
if isinstance(self.version, dict) and not self.do_not_touch:
|
||||||
|
self.version["max"] = "<=%s" % self.pypi_version
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def parse_line(cls, line):
|
def parse_line(cls, line):
|
||||||
match = cls.RE_NAME.match(line)
|
match = cls.RE_NAME.match(line)
|
||||||
@ -180,7 +206,7 @@ class Requirement(object):
|
|||||||
version = ">=%s" % self.version[2:]
|
version = ">=%s" % self.version[2:]
|
||||||
|
|
||||||
string = "%s%s" % (self.package_name, version)
|
string = "%s%s" % (self.package_name, version)
|
||||||
if self.license:
|
if self.pypi_license:
|
||||||
# NOTE(andreykurilin): When I start implementation of this script,
|
# NOTE(andreykurilin): When I start implementation of this script,
|
||||||
# python-keystoneclient dependency string took around ~45-55
|
# python-keystoneclient dependency string took around ~45-55
|
||||||
# chars, so let's use this length as indent. Feel free to modify
|
# chars, so let's use this length as indent. Feel free to modify
|
||||||
@ -190,7 +216,7 @@ class Requirement(object):
|
|||||||
indent = magic_number - len(string)
|
indent = magic_number - len(string)
|
||||||
else:
|
else:
|
||||||
indent = 2
|
indent = 2
|
||||||
string += " " * indent + "# " + self.license
|
string += " " * indent + "# " + self.pypi_license
|
||||||
return string
|
return string
|
||||||
|
|
||||||
def __eq__(self, other):
|
def __eq__(self, other):
|
||||||
@ -201,7 +227,35 @@ class Requirement(object):
|
|||||||
return not self.__eq__(other)
|
return not self.__eq__(other)
|
||||||
|
|
||||||
|
|
||||||
def parse_data(raw_data, include_comments=True):
|
class UpperConstraint(PYPIPackage):
|
||||||
|
|
||||||
|
RE_LINE = re.compile(
|
||||||
|
r"(?P<package_name>[a-zA-Z0-9-._]+)===(?P<version>[a-zA-Z0-9.]+)")
|
||||||
|
|
||||||
|
def __init__(self, package_name, version=None):
|
||||||
|
super(UpperConstraint, self).__init__(package_name)
|
||||||
|
self._version = version
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return "%s===%s" % (self.package_name, self.version)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def version(self):
|
||||||
|
if self._version is None:
|
||||||
|
self._version = self.pypi_version
|
||||||
|
return self._version
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def parse_line(cls, line):
|
||||||
|
match = cls.RE_LINE.match(line)
|
||||||
|
if match:
|
||||||
|
return cls(**match.groupdict())
|
||||||
|
|
||||||
|
def update(self, version):
|
||||||
|
self._version = version
|
||||||
|
|
||||||
|
|
||||||
|
def parse_data(raw_data, include_comments=True, dependency_cls=Requirement):
|
||||||
# first elem is None to simplify checks of last elem in requirements
|
# first elem is None to simplify checks of last elem in requirements
|
||||||
requirements = [None]
|
requirements = [None]
|
||||||
for line in raw_data.split("\n"):
|
for line in raw_data.split("\n"):
|
||||||
@ -222,13 +276,15 @@ def parse_data(raw_data, include_comments=True):
|
|||||||
if (isinstance(requirements[-1], Comment) and
|
if (isinstance(requirements[-1], Comment) and
|
||||||
not requirements[-1].is_finished):
|
not requirements[-1].is_finished):
|
||||||
requirements[-1].finish_him()
|
requirements[-1].finish_him()
|
||||||
|
|
||||||
# parse_line
|
# parse_line
|
||||||
req = Requirement.parse_line(line)
|
dep = dependency_cls.parse_line(line)
|
||||||
if req:
|
if dep:
|
||||||
if (isinstance(requirements[-1], Comment) and
|
if (isinstance(requirements[-1], Comment) and
|
||||||
DO_NOT_TOUCH_TAG in str(requirements[-1])):
|
DO_NOT_TOUCH_TAG in str(requirements[-1])):
|
||||||
req.do_not_touch = True
|
dep.do_not_touch = True
|
||||||
requirements.append(req)
|
requirements.append(dep)
|
||||||
|
|
||||||
for i in range(len(requirements) - 1, 0, -1):
|
for i in range(len(requirements) - 1, 0, -1):
|
||||||
# remove empty lines at the end of file
|
# remove empty lines at the end of file
|
||||||
if isinstance(requirements[i], Comment):
|
if isinstance(requirements[i], Comment):
|
||||||
@ -236,22 +292,27 @@ def parse_data(raw_data, include_comments=True):
|
|||||||
requirements.pop(i)
|
requirements.pop(i)
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
return requirements[1:]
|
return collections.OrderedDict(
|
||||||
|
(v if isinstance(v, Comment) else v.package_name, v)
|
||||||
|
for v in requirements if v)
|
||||||
|
|
||||||
|
|
||||||
def _read_requirements():
|
def _fetch_from_gr(filename):
|
||||||
"""Read all rally requirements."""
|
"""Try to fetch data from OpenStack global-requirements repo"""
|
||||||
LOG.info("Reading rally requirements...")
|
for i in range(0, len(GLOBAL_REQUIREMENTS_LOCATIONS)):
|
||||||
for file_name in RALLY_REQUIREMENTS_FILES:
|
url = GLOBAL_REQUIREMENTS_LOCATIONS[i] + filename
|
||||||
LOG.debug("Try to read '%s'." % file_name)
|
LOG.debug("Try to obtain %s from %s" % (filename, url))
|
||||||
with open(file_name) as f:
|
try:
|
||||||
data = f.read()
|
return requests.get(url).text
|
||||||
LOG.info("Parsing requirements from %s." % file_name)
|
except requests.ConnectionError as e:
|
||||||
yield file_name, parse_data(data)
|
LOG.exception(e)
|
||||||
|
raise Exception("Unable to obtain %s" % filename)
|
||||||
|
|
||||||
|
|
||||||
def _write_requirements(filename, requirements):
|
def _write_requirements(filename, requirements):
|
||||||
"""Saves requirements to file."""
|
"""Saves requirements to file."""
|
||||||
|
if isinstance(requirements, dict):
|
||||||
|
requirements = requirements.values()
|
||||||
LOG.info("Saving requirements to %s." % filename)
|
LOG.info("Saving requirements to %s." % filename)
|
||||||
with open(filename, "w") as f:
|
with open(filename, "w") as f:
|
||||||
for entity in requirements:
|
for entity in requirements:
|
||||||
@ -259,82 +320,66 @@ def _write_requirements(filename, requirements):
|
|||||||
f.write("\n")
|
f.write("\n")
|
||||||
|
|
||||||
|
|
||||||
def _sync():
|
def sync_requirements():
|
||||||
LOG.info("Obtaining global-requirements...")
|
|
||||||
for i in range(0, len(GLOBAL_REQUIREMENTS_LOCATIONS)):
|
|
||||||
url = GLOBAL_REQUIREMENTS_LOCATIONS[i] + GLOBAL_REQUIREMENTS_FILENAME
|
|
||||||
LOG.debug("Try to obtain global-requirements from %s" % url)
|
|
||||||
try:
|
|
||||||
raw_gr = requests.get(url).text
|
|
||||||
except requests.ConnectionError as e:
|
|
||||||
LOG.exception(e)
|
|
||||||
if i == len(GLOBAL_REQUIREMENTS_LOCATIONS) - 1:
|
|
||||||
# there are no more urls to try
|
|
||||||
raise Exception("Unable to obtain %s" %
|
|
||||||
GLOBAL_REQUIREMENTS_FILENAME)
|
|
||||||
else:
|
|
||||||
break
|
|
||||||
|
|
||||||
LOG.info("Parsing global-requirements...")
|
|
||||||
# NOTE(andreykurilin): global-requirements includes comments which can be
|
|
||||||
# unrelated to Rally project.
|
|
||||||
gr = parse_data(raw_gr, include_comments=False)
|
|
||||||
for filename, requirements in _read_requirements():
|
|
||||||
for i in range(0, len(requirements)):
|
|
||||||
if (isinstance(requirements[i], Requirement) and
|
|
||||||
not requirements[i].do_not_touch):
|
|
||||||
try:
|
|
||||||
gr_item = gr[gr.index(requirements[i])]
|
|
||||||
except ValueError:
|
|
||||||
# it not g-r requirements
|
|
||||||
if isinstance(requirements[i].version, dict):
|
|
||||||
requirements[i].version["max"] = None
|
|
||||||
else:
|
|
||||||
requirements[i].version = gr_item.version
|
|
||||||
yield filename, requirements
|
|
||||||
|
|
||||||
|
|
||||||
def sync():
|
|
||||||
"""Synchronizes Rally requirements with OpenStack global-requirements."""
|
"""Synchronizes Rally requirements with OpenStack global-requirements."""
|
||||||
for filename, requirements in _sync():
|
LOG.info("Obtaining global-requirements of OpenStack...")
|
||||||
_write_requirements(filename, requirements)
|
raw_gr = _fetch_from_gr("global-requirements.txt")
|
||||||
|
|
||||||
|
# NOTE(andreykurilin): global-requirements includes comments which can be
|
||||||
|
# unrelated to Rally project, so let's just ignore them
|
||||||
|
gr = parse_data(raw_gr, include_comments=False)
|
||||||
|
for file_name in RALLY_REQUIREMENTS_FILES:
|
||||||
|
LOG.debug("Processing '%s'." % file_name)
|
||||||
|
with open(file_name) as f:
|
||||||
|
requirements = parse_data(f.read())
|
||||||
|
for name, req in requirements.items():
|
||||||
|
if isinstance(req, Requirement) and not req.do_not_touch:
|
||||||
|
if name in gr:
|
||||||
|
req.version = gr[req.package_name].version
|
||||||
|
else:
|
||||||
|
# it not g-r requirements
|
||||||
|
if isinstance(req.version, dict):
|
||||||
|
req.version["max"] = None
|
||||||
|
_write_requirements(file_name, requirements)
|
||||||
|
|
||||||
|
|
||||||
def format_requirements():
|
def update_upper_constraints():
|
||||||
"""Obtain package licenses from pypy and write requirements to file."""
|
"""Obtains latest version of packages and put them to upper-constraints."""
|
||||||
for filename, requirements in _read_requirements():
|
LOG.info("Obtaining upper-constrains from OpenStack...")
|
||||||
_write_requirements(filename, requirements)
|
raw_g_uc = _fetch_from_gr("upper-constraints.txt")
|
||||||
|
# NOTE(andreykurilin): global OpenStack upper-constraints file includes
|
||||||
|
# comments which can be unrelated to Rally project, so let's just ignore
|
||||||
|
# them.
|
||||||
|
global_uc = parse_data(raw_g_uc, include_comments=False,
|
||||||
|
dependency_cls=UpperConstraint)
|
||||||
|
with open("upper-constraints.txt") as f:
|
||||||
|
our_uc = parse_data(f.read(), dependency_cls=UpperConstraint)
|
||||||
|
with open("requirements.txt") as f:
|
||||||
|
our_requirements = parse_data(f.read(), include_comments=False)
|
||||||
|
|
||||||
|
for name, req in our_requirements.items():
|
||||||
|
if isinstance(req, Comment):
|
||||||
|
print("continue")
|
||||||
|
continue
|
||||||
|
print(req)
|
||||||
|
if name not in our_uc:
|
||||||
|
our_uc[name] = UpperConstraint(name)
|
||||||
|
|
||||||
def add_uppers():
|
if name in global_uc:
|
||||||
"""Obtains latest version of packages and put them to requirements."""
|
# we cannot use whatever we want versions in CI. OpenStack CI
|
||||||
for filename, requirements in _sync():
|
# installs ignores versions listed in requirements of
|
||||||
LOG.info("Obtaining latest versions of packages for %s." % filename)
|
# particular project and use versions from global u-c file.
|
||||||
for req in requirements:
|
# It means that we need to suggest to use the same versions
|
||||||
if isinstance(req, Requirement):
|
our_uc[name].update(global_uc[name].version)
|
||||||
if isinstance(req.version, dict) and not req.version["max"]:
|
|
||||||
req.sync_max_version_with_pypy()
|
our_uc = sorted(our_uc.values(), key=lambda o: o.package_name.upper())
|
||||||
_write_requirements(filename, requirements)
|
_write_requirements("upper-constraints.txt", our_uc)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(
|
sync_requirements()
|
||||||
prog="Python Requirement Manager for Rally",
|
update_upper_constraints()
|
||||||
description=__doc__.strip(),
|
|
||||||
add_help=True
|
|
||||||
)
|
|
||||||
|
|
||||||
action_groups = parser.add_mutually_exclusive_group()
|
|
||||||
action_groups.add_argument("--format",
|
|
||||||
action="store_const",
|
|
||||||
const=format_requirements,
|
|
||||||
dest="action")
|
|
||||||
action_groups.add_argument("--add-upper",
|
|
||||||
action="store_const",
|
|
||||||
const=add_uppers,
|
|
||||||
dest="action")
|
|
||||||
action_groups.set_defaults(action=sync)
|
|
||||||
parser.parse_args(sys.argv[1:]).action()
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
sys.exit(main())
|
sys.exit(main())
|
||||||
|
45
upper-constraints.txt
Normal file
45
upper-constraints.txt
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
alembic===0.9.6
|
||||||
|
boto===2.48.0
|
||||||
|
decorator===4.1.2
|
||||||
|
gnocchiclient===4.0.0
|
||||||
|
Jinja2===2.9.6
|
||||||
|
jsonschema===2.6.0
|
||||||
|
keystoneauth1===3.2.0
|
||||||
|
kubernetes===2.0.0
|
||||||
|
morph===0.1.2
|
||||||
|
netaddr===0.7.19
|
||||||
|
os-faults===0.1.17
|
||||||
|
oslo.config===5.0.0
|
||||||
|
oslo.db===4.29.0
|
||||||
|
oslo.log===3.32.0
|
||||||
|
oslo.utils===3.30.0
|
||||||
|
osprofiler===1.13.0
|
||||||
|
paramiko===2.3.1
|
||||||
|
pbr===3.1.1
|
||||||
|
PrettyTable===7
|
||||||
|
python-ceilometerclient===2.9.0
|
||||||
|
python-cinderclient===3.2.0
|
||||||
|
python-designateclient===2.7.0
|
||||||
|
python-glanceclient===2.8.0
|
||||||
|
python-heatclient===1.12.0
|
||||||
|
python-ironicclient===1.17.1
|
||||||
|
python-keystoneclient===3.13.0
|
||||||
|
python-magnumclient===2.7.0
|
||||||
|
python-manilaclient===1.17.2
|
||||||
|
python-mistralclient===3.1.3
|
||||||
|
python-monascaclient===1.8.0
|
||||||
|
python-muranoclient===0.14.0
|
||||||
|
python-neutronclient===6.5.0
|
||||||
|
python-novaclient===9.1.1
|
||||||
|
python-saharaclient===1.3.0
|
||||||
|
python-senlinclient===1.4.0
|
||||||
|
python-subunit===1.2.0
|
||||||
|
python-swiftclient===3.4.0
|
||||||
|
python-troveclient===2.12.0
|
||||||
|
python-watcherclient===1.4.0
|
||||||
|
python-zaqarclient===1.7.0
|
||||||
|
PyYAML===3.12
|
||||||
|
requests===2.18.4
|
||||||
|
six===1.11.0
|
||||||
|
SQLAlchemy===1.1.14
|
||||||
|
virtualenv===15.1.0
|
Loading…
Reference in New Issue
Block a user