[requirements] Add helper for managing requirements
While we are not depend on OpenStack global-requirements, we need to be synced
with it to be sure that Rally compatible with all OpenStack related stuff.
This patch provides new helper script, which checks versions from g-r, correct
license and latest version from pypi.
The usage is simple:
# sync rally requirements with g-r
tox -e requirements
# just format rally requirements
tox -e requirements -- --format
# sync rally requirements with g-r & add upper limits for all packages
tox -e requirements -- --add-uppers
Also, this patch formats Rally requirements in unified form, puts right
licenses to requirements and adds rally licenses in correct way in setup.cfg
Change-Id: Ia50694f636d9f80f08d90cc8133ebac8bb3f8671
This commit is contained in:
Andrey Kurilin
parent
f0adca5c4b
commit
991d5b4abb
|
|
@ -1,40 +1,44 @@
|
|||
# The order of packages is significant, because pip processes them in the order
|
||||
# of appearance. Changing the order has an impact on the overall integration
|
||||
# process, which may cause wedges in the gate later.
|
||||
alembic>=0.8.4 # MIT
|
||||
boto>=2.32.1 # MIT
|
||||
decorator>=3.4.0 # BSD
|
||||
Jinja2>=2.8 # BSD License (3 clause)
|
||||
jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT
|
||||
netaddr!=0.7.16,>=0.7.12 # BSD
|
||||
oslo.config>=3.10.0 # Apache-2.0
|
||||
oslo.db>=4.1.0 # Apache-2.0
|
||||
oslo.i18n>=2.1.0 # Apache-2.0
|
||||
oslo.log>=1.14.0 # Apache-2.0
|
||||
oslo.serialization>=1.10.0 # Apache-2.0
|
||||
oslo.utils>=3.11.0 # Apache-2.0
|
||||
paramiko>=2.0 # LGPL
|
||||
pbr>=1.6 # Apache-2.0
|
||||
PrettyTable<0.8,>=0.7 # BSD
|
||||
PyYAML>=3.1.0 # MIT
|
||||
python-designateclient>=1.5.0 # Apache-2.0
|
||||
python-glanceclient>=2.0.0 # Apache-2.0
|
||||
python-keystoneclient!=1.8.0,!=2.1.0,>=1.7.0 # Apache-2.0
|
||||
keystoneauth1>=2.1.0 # Apache-2.0
|
||||
python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
|
||||
python-neutronclient>=4.2.0 # Apache-2.0
|
||||
python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0 # Apache-2.0
|
||||
python-manilaclient>=1.3.0 # Apache-2.0
|
||||
python-heatclient>=1.1.0 # Apache-2.0
|
||||
python-ceilometerclient>=2.2.1 # Apache-2.0
|
||||
python-ironicclient>=1.1.0 # Apache-2.0
|
||||
python-saharaclient>=0.13.0 # Apache-2.0
|
||||
python-troveclient>=2.2.0 # Apache-2.0
|
||||
python-zaqarclient>=1.0.0 # Apache-2.0
|
||||
python-swiftclient>=2.2.0 # Apache-2.0
|
||||
python-watcherclient>=0.23.0 # Apache-2.0
|
||||
python-subunit>=0.0.18 # Apache-2.0/BSD
|
||||
requests>=2.10.0 # Apache-2.0
|
||||
SQLAlchemy<1.1.0,>=1.0.10 # MIT
|
||||
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD
|
||||
six>=1.9.0 # MIT
|
||||
alembic>=0.8.4 # MIT
|
||||
boto>=2.32.1 # MIT
|
||||
decorator>=3.4.0 # new BSD License
|
||||
Jinja2>=2.8 # BSD
|
||||
# [constant-upper-limit]
|
||||
jsonschema!=2.5.0,>=2.0.0,<3.0.0 # MIT
|
||||
netaddr!=0.7.16,>=0.7.12 # BSD
|
||||
oslo.config>=3.10.0 # Apache Software License
|
||||
oslo.db>=4.1.0 # Apache Software License
|
||||
oslo.i18n>=2.1.0 # Apache Software License
|
||||
oslo.log>=1.14.0 # Apache Software License
|
||||
oslo.serialization>=1.10.0 # Apache Software License
|
||||
oslo.utils>=3.11.0 # Apache Software License
|
||||
paramiko>=2.0 # LGPL
|
||||
pbr>=1.6 # Apache Software License
|
||||
# [constant-upper-limit]
|
||||
PrettyTable>=0.7,<0.8 # BSD
|
||||
PyYAML>=3.1.0 # MIT
|
||||
python-designateclient>=1.5.0 # Apache License, Version 2.0
|
||||
python-glanceclient>=2.0.0 # Apache License, Version 2.0
|
||||
python-keystoneclient!=1.8.0,!=2.1.0,>=1.7.0 # Apache Software License
|
||||
keystoneauth1>=2.1.0 # Apache Software License
|
||||
python-novaclient!=2.33.0,>=2.29.0 # Apache License, Version 2.0
|
||||
python-neutronclient>=4.2.0 # Apache Software License
|
||||
python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0 # Apache Software License
|
||||
python-manilaclient>=1.3.0 # Apache Software License
|
||||
python-heatclient>=1.1.0 # Apache Software License
|
||||
python-ceilometerclient>=2.2.1 # Apache Software License
|
||||
python-ironicclient>=1.1.0 # Apache Software License
|
||||
python-saharaclient>=0.13.0 # Apache License, Version 2.0
|
||||
python-troveclient>=2.2.0 # Apache Software License
|
||||
python-zaqarclient>=1.0.0 # Apache Software License
|
||||
python-swiftclient>=2.2.0 # Apache Software License
|
||||
python-watcherclient>=0.23.0 # Apache Software License
|
||||
python-subunit>=0.0.18
|
||||
requests>=2.10.0 # Apache License, Version 2.0
|
||||
# [constant-upper-limit]
|
||||
SQLAlchemy>=1.0.10,<1.1.0 # MIT
|
||||
# [constant-upper-limit]
|
||||
sphinx!=1.2.0,!=1.3b1,>=1.1.2,<1.3 # BSD
|
||||
six>=1.9.0 # MIT
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ description-file =
|
|||
author = OpenStack
|
||||
author-email = openstack-dev@lists.openstack.org
|
||||
home-page = http://www.openstack.org/
|
||||
license = Apache License, Version 2.0
|
||||
classifier =
|
||||
Environment :: OpenStack
|
||||
Intended Audience :: Developers
|
||||
|
|
|
|||
|
|
@ -1,16 +1,17 @@
|
|||
# The order of packages is significant, because pip processes them in the order
|
||||
# of appearance. Changing the order has an impact on the overall integration
|
||||
# process, which may cause wedges in the gate later.
|
||||
hacking<0.10,>=0.9.2
|
||||
pytest>=2.7
|
||||
pytest-cov>=2.2.1
|
||||
pytest-html
|
||||
# [constant-upper-limit]
|
||||
hacking>=0.9.2,<0.10 # Apache Software License
|
||||
pytest>=2.7 # MIT license
|
||||
pytest-cov>=2.2.1 # MIT
|
||||
pytest-html # Mozilla Public License 2.0 (MPL 2.0)
|
||||
|
||||
coverage>=3.6 # Apache-2.0
|
||||
ddt>=1.0.1 # MIT
|
||||
mock>=2.0 # BSD
|
||||
python-dateutil>=2.4.2 # BSD
|
||||
testtools>=1.4.0 # MIT
|
||||
coverage>=3.6 # Apache License, Version 2.0
|
||||
ddt>=1.0.1
|
||||
mock>=2.0
|
||||
python-dateutil>=2.4.2 # Simplified BSD
|
||||
testtools>=1.4.0
|
||||
|
||||
oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
|
||||
oslotest>=1.10.0 # Apache-2.0
|
||||
oslosphinx!=3.4.0,>=2.5.0 # Apache Software License
|
||||
oslotest>=1.10.0 # Apache Software License
|
||||
|
|
|
|||
|
|
@ -0,0 +1,317 @@
|
|||
#!/usr/bin/env python
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Synchronizes, formats and prepares requirements to release(obtains and adds
|
||||
maximum allowed version).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
if not LOG.handlers:
|
||||
LOG.addHandler(logging.StreamHandler())
|
||||
LOG.setLevel(logging.INFO)
|
||||
|
||||
|
||||
GLOBAL_REQUIREMENTS_LOCATIONS = (
|
||||
"https://raw.githubusercontent.com/openstack/requirements/master/",
|
||||
"http://git.openstack.org/cgit/openstack/requirements/plain/"
|
||||
)
|
||||
GLOBAL_REQUIREMENTS_FILENAME = "global-requirements.txt"
|
||||
RALLY_REQUIREMENTS_FILES = (
|
||||
"requirements.txt",
|
||||
"test-requirements.txt",
|
||||
"optional-requirements.txt"
|
||||
)
|
||||
UPPER_LIMIT_TAG = "[constant-upper-limit]"
|
||||
|
||||
|
||||
class Comment(object):
|
||||
def __init__(self, s=None):
|
||||
self._comments = []
|
||||
self.is_finished = False
|
||||
if s:
|
||||
self.append(s)
|
||||
|
||||
def finish_him(self):
|
||||
self.is_finished = True
|
||||
|
||||
def append(self, s):
|
||||
self._comments.append(s[1:].strip())
|
||||
|
||||
def __str__(self):
|
||||
return textwrap.fill("\n".join(self._comments), width=80,
|
||||
initial_indent="# ", subsequent_indent="# ")
|
||||
|
||||
|
||||
class Requirement(object):
|
||||
RE_NAME = re.compile(r"[a-zA-Z0-9-._]+")
|
||||
RE_CONST_VERSION = re.compile(r"==[a-zA-Z0-9.]+")
|
||||
RE_MIN_VERSION = re.compile(r">=?[a-zA-Z0-9.]+")
|
||||
RE_MAX_VERSION = re.compile(r"<=?[a-zA-Z0-9.]+")
|
||||
RE_NE_VERSIONS = re.compile(r"!=[a-zA-Z0-9.]+")
|
||||
# NOTE(andreykurilin): one license can have different labels. Let's use
|
||||
# unified variant.
|
||||
LICENSE_MAP = {"MIT license": "MIT",
|
||||
"MIT License": "MIT",
|
||||
"BSD License": "BSD",
|
||||
"Apache 2.0": "Apache License, Version 2.0"}
|
||||
|
||||
def __init__(self, package_name, version):
|
||||
self.package_name = package_name
|
||||
self.version = version
|
||||
self._license = None
|
||||
self._pypy_info = None
|
||||
self.upper_limit_flag = False
|
||||
|
||||
def sync_max_version_with_pypy(self):
|
||||
if isinstance(self.version, dict) and not self.upper_limit_flag:
|
||||
self.version["max"] = "<=%s" % self.pypy_info["info"]["version"]
|
||||
|
||||
@property
|
||||
def pypy_info(self):
|
||||
if self._pypy_info is None:
|
||||
resp = requests.get("https://pypi.python.org/pypi/%s/json" %
|
||||
self.package_name)
|
||||
if resp.status_code != 200:
|
||||
raise Exception(resp.text)
|
||||
self._pypy_info = resp.json()
|
||||
return self._pypy_info
|
||||
|
||||
@property
|
||||
def license(self):
|
||||
if self._license is None:
|
||||
if self.pypy_info["info"]["license"]:
|
||||
self._license = self.pypy_info["info"]["license"]
|
||||
else:
|
||||
# try to parse classifiers
|
||||
prefix = "License :: OSI Approved :: "
|
||||
classifiers = [c[len(prefix):]
|
||||
for c in self.pypy_info["info"]["classifiers"]
|
||||
if c.startswith(prefix)]
|
||||
self._license = "/".join(classifiers)
|
||||
self._license = self.LICENSE_MAP.get(self._license, self._license)
|
||||
if self._license == "UNKNOWN":
|
||||
self._license = None
|
||||
return self._license
|
||||
|
||||
@classmethod
|
||||
def parse_line(cls, line):
|
||||
match = cls.RE_NAME.match(line)
|
||||
if match:
|
||||
name = match.group()
|
||||
# remove name
|
||||
versions = line.replace(name, "")
|
||||
# remove comments
|
||||
versions = versions.split("#")[0]
|
||||
# remove python classifiers
|
||||
versions = versions.split(";")[0].strip()
|
||||
if not cls.RE_CONST_VERSION.match(versions):
|
||||
versions = versions.strip().split(",")
|
||||
min_version = None
|
||||
max_version = None
|
||||
ne_versions = []
|
||||
for version in versions:
|
||||
if cls.RE_MIN_VERSION.match(version):
|
||||
if min_version:
|
||||
raise Exception("Found several min versions for "
|
||||
"%s package." % name)
|
||||
min_version = version
|
||||
elif cls.RE_MAX_VERSION.match(version):
|
||||
if max_version:
|
||||
raise Exception("Found several max versions for "
|
||||
"%s package." % name)
|
||||
max_version = version
|
||||
elif cls.RE_NE_VERSIONS.match(version):
|
||||
ne_versions.append(version)
|
||||
versions = {"min": min_version,
|
||||
"max": max_version,
|
||||
"ne": ne_versions}
|
||||
return cls(name, versions)
|
||||
|
||||
def __str__(self):
|
||||
if isinstance(self.version, dict):
|
||||
version = ""
|
||||
if self.version["ne"]:
|
||||
version += ",".join(self.version["ne"])
|
||||
if self.version["min"]:
|
||||
if version:
|
||||
version += ","
|
||||
version += self.version["min"]
|
||||
if self.version["max"]:
|
||||
if version:
|
||||
version += ","
|
||||
version += self.version["max"]
|
||||
else:
|
||||
version = self.version
|
||||
|
||||
string = "%s%s" % (self.package_name, version)
|
||||
if self.license:
|
||||
# NOTE(andreykurilin): When I start implementation of this script,
|
||||
# python-keystoneclient dependency took around ~45-55, so let's
|
||||
# use this length as indent. Feel free to modify it to lower or
|
||||
# greater value.
|
||||
magic_number = 55
|
||||
if len(string) < magic_number:
|
||||
indent = magic_number - len(string)
|
||||
else:
|
||||
indent = 2
|
||||
string += " " * indent + "# " + self.license
|
||||
return string
|
||||
|
||||
def __eq__(self, other):
|
||||
return (isinstance(other, self.__class__) and
|
||||
self.package_name == other.package_name)
|
||||
|
||||
|
||||
def parse_data(raw_data, include_comments=True):
|
||||
# first elem is None to simplify checks of last elem in requirements
|
||||
requirements = [None]
|
||||
for line in raw_data.split("\n"):
|
||||
if line.startswith("#"):
|
||||
if not include_comments:
|
||||
continue
|
||||
|
||||
if (isinstance(requirements[-1], Comment) and
|
||||
UPPER_LIMIT_TAG in line):
|
||||
requirements[-1].is_finished = True
|
||||
|
||||
if getattr(requirements[-1], "is_finished", True):
|
||||
requirements.append(Comment())
|
||||
|
||||
requirements[-1].append(line)
|
||||
else:
|
||||
if (isinstance(requirements[-1], Comment) and
|
||||
not requirements[-1].is_finished):
|
||||
requirements[-1].finish_him()
|
||||
if line == "":
|
||||
requirements.append("")
|
||||
else:
|
||||
# parse_line
|
||||
req = Requirement.parse_line(line)
|
||||
if req:
|
||||
if (isinstance(requirements[-1], Comment) and
|
||||
str(requirements[-1]).endswith(UPPER_LIMIT_TAG)):
|
||||
req.upper_limit_flag = True
|
||||
requirements.append(req)
|
||||
if not requirements[-1]:
|
||||
requirements.pop()
|
||||
return requirements[1:]
|
||||
|
||||
|
||||
def _read_requirements():
|
||||
"""Read all rally requirements."""
|
||||
LOG.info("Reading rally requirements...")
|
||||
for file_name in RALLY_REQUIREMENTS_FILES:
|
||||
LOG.debug("Try to read '%s'." % file_name)
|
||||
with open(file_name) as f:
|
||||
data = f.read()
|
||||
LOG.info("Parsing requirements from %s." % file_name)
|
||||
yield file_name, parse_data(data)
|
||||
|
||||
|
||||
def _write_requirements(filename, requirements):
|
||||
"""Saves requirements to file."""
|
||||
LOG.info("Saving requirements to %s." % filename)
|
||||
with open(filename, "w") as f:
|
||||
for entity in requirements:
|
||||
f.write(str(entity))
|
||||
f.write("\n")
|
||||
|
||||
|
||||
def _sync():
|
||||
LOG.info("Obtaining global-requirements...")
|
||||
for i in range(0, len(GLOBAL_REQUIREMENTS_LOCATIONS)):
|
||||
url = GLOBAL_REQUIREMENTS_LOCATIONS[i] + GLOBAL_REQUIREMENTS_FILENAME
|
||||
LOG.debug("Try to obtain global-requirements from %s" % url)
|
||||
try:
|
||||
raw_gr = requests.get(url).text
|
||||
except requests.ConnectionError as e:
|
||||
LOG.exception(e)
|
||||
if i == len(GLOBAL_REQUIREMENTS_LOCATIONS) - 1:
|
||||
# there are no more urls to try
|
||||
raise Exception("Unable to obtain %s" %
|
||||
GLOBAL_REQUIREMENTS_FILENAME)
|
||||
else:
|
||||
break
|
||||
|
||||
LOG.info("Parsing global-requirements...")
|
||||
# NOTE(andreykurilin): global-requirements includes comments which can be
|
||||
# unrelated to Rally project.
|
||||
gr = parse_data(raw_gr, include_comments=False)
|
||||
for filename, requirements in _read_requirements():
|
||||
for i in range(0, len(requirements)):
|
||||
if isinstance(requirements[i], Requirement):
|
||||
try:
|
||||
gr_item = gr[gr.index(requirements[i])]
|
||||
except ValueError:
|
||||
# it not g-r requirements
|
||||
pass
|
||||
else:
|
||||
requirements[i].version = gr_item.version
|
||||
yield filename, requirements
|
||||
|
||||
|
||||
def sync():
|
||||
"""Synchronizes Rally requirements with OpenStack global-requirements."""
|
||||
for filename, requirements in _sync():
|
||||
_write_requirements(filename, requirements)
|
||||
|
||||
|
||||
def format_requirements():
|
||||
"""Obtain package licenses from pypy and write requirements to file."""
|
||||
for filename, requirements in _read_requirements():
|
||||
_write_requirements(filename, requirements)
|
||||
|
||||
|
||||
def add_uppers():
|
||||
"""Obtains latest version of packages and put them to requirements."""
|
||||
for filename, requirements in _sync():
|
||||
LOG.info("Obtaining latest versions of packages from %s." % filename)
|
||||
for req in requirements:
|
||||
if isinstance(req, Requirement):
|
||||
req.sync_max_version_with_pypy()
|
||||
_write_requirements(filename, requirements)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="Python Requirement Manager for Rally",
|
||||
description=__doc__.strip(),
|
||||
add_help=True
|
||||
)
|
||||
|
||||
action_groups = parser.add_mutually_exclusive_group()
|
||||
action_groups.add_argument("--format",
|
||||
action="store_const",
|
||||
const=format_requirements,
|
||||
dest="action")
|
||||
action_groups.add_argument("--add-upper",
|
||||
action="store_const",
|
||||
const=add_uppers,
|
||||
dest="action")
|
||||
action_groups.set_defaults(action=sync)
|
||||
parser.parse_args(sys.argv[1:]).action()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -154,7 +154,8 @@ def check_import_of_logging(logical_line, physical_line, filename):
|
|||
|
||||
excluded_files = ["./rally/common/logging.py",
|
||||
"./tests/unit/test_logging.py",
|
||||
"./tests/ci/rally_verify.py"]
|
||||
"./tests/ci/rally_verify.py",
|
||||
"./tests/ci/sync_requirements.py"]
|
||||
|
||||
forbidden_imports = ["from oslo_log",
|
||||
"import oslo_log",
|
||||
|
|
|
|||
Loading…
Reference in New Issue