[requirements] Add helper for managing requirements

While we are not depend on OpenStack global-requirements, we need to be synced
with it to be sure that Rally compatible with all OpenStack related stuff.

This patch provides new helper script, which checks versions from g-r, correct
license and latest version from pypi.

The usage is simple:

   # sync rally requirements with g-r
   tox -e requirements

   # just format rally requirements
   tox -e requirements -- --format

    # sync rally requirements with g-r & add upper limits for all packages
   tox -e requirements -- --add-uppers

Also, this patch formats Rally requirements in unified form, puts right
licenses to requirements and adds rally licenses in correct way in setup.cfg

Change-Id: Ia50694f636d9f80f08d90cc8133ebac8bb3f8671

requirements.txt

@@ -3,38 +3,42 @@
 # process, which may cause wedges in the gate later.
 alembic>=0.8.4                                         # MIT
 boto>=2.32.1                                           # MIT
-decorator>=3.4.0 # BSD
+decorator>=3.4.0                                       # new BSD License
-Jinja2>=2.8 # BSD License (3 clause)
+Jinja2>=2.8                                            # BSD
-jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT
+# [constant-upper-limit]
+jsonschema!=2.5.0,>=2.0.0,<3.0.0                       # MIT
 netaddr!=0.7.16,>=0.7.12                               # BSD
-oslo.config>=3.10.0 # Apache-2.0
+oslo.config>=3.10.0                                    # Apache Software License
-oslo.db>=4.1.0 # Apache-2.0
+oslo.db>=4.1.0                                         # Apache Software License
-oslo.i18n>=2.1.0 # Apache-2.0
+oslo.i18n>=2.1.0                                       # Apache Software License
-oslo.log>=1.14.0 # Apache-2.0
+oslo.log>=1.14.0                                       # Apache Software License
-oslo.serialization>=1.10.0 # Apache-2.0
+oslo.serialization>=1.10.0                             # Apache Software License
-oslo.utils>=3.11.0 # Apache-2.0
+oslo.utils>=3.11.0                                     # Apache Software License
 paramiko>=2.0                                          # LGPL
-pbr>=1.6 # Apache-2.0
+pbr>=1.6                                               # Apache Software License
-PrettyTable<0.8,>=0.7 # BSD
+# [constant-upper-limit]
+PrettyTable>=0.7,<0.8                                  # BSD
 PyYAML>=3.1.0                                          # MIT
-python-designateclient>=1.5.0 # Apache-2.0
+python-designateclient>=1.5.0                          # Apache License, Version 2.0
-python-glanceclient>=2.0.0 # Apache-2.0
+python-glanceclient>=2.0.0                             # Apache License, Version 2.0
-python-keystoneclient!=1.8.0,!=2.1.0,>=1.7.0 # Apache-2.0
+python-keystoneclient!=1.8.0,!=2.1.0,>=1.7.0           # Apache Software License
-keystoneauth1>=2.1.0 # Apache-2.0
+keystoneauth1>=2.1.0                                   # Apache Software License
-python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
+python-novaclient!=2.33.0,>=2.29.0                     # Apache License, Version 2.0
-python-neutronclient>=4.2.0 # Apache-2.0
+python-neutronclient>=4.2.0                            # Apache Software License
-python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0 # Apache-2.0
+python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0             # Apache Software License
-python-manilaclient>=1.3.0 # Apache-2.0
+python-manilaclient>=1.3.0                             # Apache Software License
-python-heatclient>=1.1.0 # Apache-2.0
+python-heatclient>=1.1.0                               # Apache Software License
-python-ceilometerclient>=2.2.1 # Apache-2.0
+python-ceilometerclient>=2.2.1                         # Apache Software License
-python-ironicclient>=1.1.0 # Apache-2.0
+python-ironicclient>=1.1.0                             # Apache Software License
-python-saharaclient>=0.13.0 # Apache-2.0
+python-saharaclient>=0.13.0                            # Apache License, Version 2.0
-python-troveclient>=2.2.0 # Apache-2.0
+python-troveclient>=2.2.0                              # Apache Software License
-python-zaqarclient>=1.0.0 # Apache-2.0
+python-zaqarclient>=1.0.0                              # Apache Software License
-python-swiftclient>=2.2.0 # Apache-2.0
+python-swiftclient>=2.2.0                              # Apache Software License
-python-watcherclient>=0.23.0 # Apache-2.0
+python-watcherclient>=0.23.0                           # Apache Software License
-python-subunit>=0.0.18 # Apache-2.0/BSD
+python-subunit>=0.0.18
-requests>=2.10.0 # Apache-2.0
+requests>=2.10.0                                       # Apache License, Version 2.0
-SQLAlchemy<1.1.0,>=1.0.10 # MIT
+# [constant-upper-limit]
-sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD
+SQLAlchemy>=1.0.10,<1.1.0                              # MIT
+# [constant-upper-limit]
+sphinx!=1.2.0,!=1.3b1,>=1.1.2,<1.3                     # BSD
 six>=1.9.0                                             # MIT

setup.cfg

@@ -6,6 +6,7 @@ description-file =
 author = OpenStack
 author-email = openstack-dev@lists.openstack.org
 home-page = http://www.openstack.org/
+license = Apache License, Version 2.0
 classifier =
     Environment :: OpenStack
     Intended Audience :: Developers

test-requirements.txt

@@ -1,16 +1,17 @@
 # The order of packages is significant, because pip processes them in the order
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
-hacking<0.10,>=0.9.2
+# [constant-upper-limit]
-pytest>=2.7
+hacking>=0.9.2,<0.10                                   # Apache Software License
-pytest-cov>=2.2.1
+pytest>=2.7                                            # MIT license
-pytest-html
+pytest-cov>=2.2.1                                      # MIT
+pytest-html                                            # Mozilla Public License 2.0 (MPL 2.0)
 
-coverage>=3.6 # Apache-2.0
+coverage>=3.6                                          # Apache License, Version 2.0
-ddt>=1.0.1 # MIT
+ddt>=1.0.1
-mock>=2.0 # BSD
+mock>=2.0
-python-dateutil>=2.4.2 # BSD
+python-dateutil>=2.4.2                                 # Simplified BSD
-testtools>=1.4.0 # MIT
+testtools>=1.4.0
 
-oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
+oslosphinx!=3.4.0,>=2.5.0                              # Apache Software License
-oslotest>=1.10.0 # Apache-2.0
+oslotest>=1.10.0                                       # Apache Software License

tests/ci/sync_requirements.py

@@ -0,0 +1,317 @@
+#!/usr/bin/env python
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Synchronizes, formats and prepares requirements to release(obtains and adds
+maximum allowed version).
+"""
+
+import argparse
+import logging
+import re
+import sys
+import textwrap
+
+import requests
+
+
+LOG = logging.getLogger(__name__)
+if not LOG.handlers:
+    LOG.addHandler(logging.StreamHandler())
+    LOG.setLevel(logging.INFO)
+
+
+GLOBAL_REQUIREMENTS_LOCATIONS = (
+    "https://raw.githubusercontent.com/openstack/requirements/master/",
+    "http://git.openstack.org/cgit/openstack/requirements/plain/"
+)
+GLOBAL_REQUIREMENTS_FILENAME = "global-requirements.txt"
+RALLY_REQUIREMENTS_FILES = (
+    "requirements.txt",
+    "test-requirements.txt",
+    "optional-requirements.txt"
+)
+UPPER_LIMIT_TAG = "[constant-upper-limit]"
+
+
+class Comment(object):
+    def __init__(self, s=None):
+        self._comments = []
+        self.is_finished = False
+        if s:
+            self.append(s)
+
+    def finish_him(self):
+        self.is_finished = True
+
+    def append(self, s):
+        self._comments.append(s[1:].strip())
+
+    def __str__(self):
+        return textwrap.fill("\n".join(self._comments), width=80,
+                             initial_indent="# ", subsequent_indent="# ")
+
+
+class Requirement(object):
+    RE_NAME = re.compile(r"[a-zA-Z0-9-._]+")
+    RE_CONST_VERSION = re.compile(r"==[a-zA-Z0-9.]+")
+    RE_MIN_VERSION = re.compile(r">=?[a-zA-Z0-9.]+")
+    RE_MAX_VERSION = re.compile(r"<=?[a-zA-Z0-9.]+")
+    RE_NE_VERSIONS = re.compile(r"!=[a-zA-Z0-9.]+")
+    # NOTE(andreykurilin): one license can have different labels. Let's use
+    #   unified variant.
+    LICENSE_MAP = {"MIT license": "MIT",
+                   "MIT License": "MIT",
+                   "BSD License": "BSD",
+                   "Apache 2.0": "Apache License, Version 2.0"}
+
+    def __init__(self, package_name, version):
+        self.package_name = package_name
+        self.version = version
+        self._license = None
+        self._pypy_info = None
+        self.upper_limit_flag = False
+
+    def sync_max_version_with_pypy(self):
+        if isinstance(self.version, dict) and not self.upper_limit_flag:
+            self.version["max"] = "<=%s" % self.pypy_info["info"]["version"]
+
+    @property
+    def pypy_info(self):
+        if self._pypy_info is None:
+            resp = requests.get("https://pypi.python.org/pypi/%s/json" %
+                                self.package_name)
+            if resp.status_code != 200:
+                raise Exception(resp.text)
+            self._pypy_info = resp.json()
+        return self._pypy_info
+
+    @property
+    def license(self):
+        if self._license is None:
+            if self.pypy_info["info"]["license"]:
+                self._license = self.pypy_info["info"]["license"]
+            else:
+                # try to parse classifiers
+                prefix = "License :: OSI Approved :: "
+                classifiers = [c[len(prefix):]
+                               for c in self.pypy_info["info"]["classifiers"]
+                               if c.startswith(prefix)]
+                self._license = "/".join(classifiers)
+            self._license = self.LICENSE_MAP.get(self._license, self._license)
+            if self._license == "UNKNOWN":
+                self._license = None
+        return self._license
+
+    @classmethod
+    def parse_line(cls, line):
+        match = cls.RE_NAME.match(line)
+        if match:
+            name = match.group()
+            # remove name
+            versions = line.replace(name, "")
+            # remove comments
+            versions = versions.split("#")[0]
+            # remove python classifiers
+            versions = versions.split(";")[0].strip()
+            if not cls.RE_CONST_VERSION.match(versions):
+                versions = versions.strip().split(",")
+                min_version = None
+                max_version = None
+                ne_versions = []
+                for version in versions:
+                    if cls.RE_MIN_VERSION.match(version):
+                        if min_version:
+                            raise Exception("Found several min versions for "
+                                            "%s package." % name)
+                        min_version = version
+                    elif cls.RE_MAX_VERSION.match(version):
+                        if max_version:
+                            raise Exception("Found several max versions for "
+                                            "%s package." % name)
+                        max_version = version
+                    elif cls.RE_NE_VERSIONS.match(version):
+                        ne_versions.append(version)
+                versions = {"min": min_version,
+                            "max": max_version,
+                            "ne": ne_versions}
+            return cls(name, versions)
+
+    def __str__(self):
+        if isinstance(self.version, dict):
+            version = ""
+            if self.version["ne"]:
+                    version += ",".join(self.version["ne"])
+            if self.version["min"]:
+                if version:
+                    version += ","
+                version += self.version["min"]
+            if self.version["max"]:
+                if version:
+                    version += ","
+                    version += self.version["max"]
+        else:
+            version = self.version
+
+        string = "%s%s" % (self.package_name, version)
+        if self.license:
+            # NOTE(andreykurilin): When I start implementation of this script,
+            #   python-keystoneclient dependency took around ~45-55, so let's
+            #   use this length as indent. Feel free to modify it to lower or
+            #   greater value.
+            magic_number = 55
+            if len(string) < magic_number:
+                indent = magic_number - len(string)
+            else:
+                indent = 2
+            string += " " * indent + "# " + self.license
+        return string
+
+    def __eq__(self, other):
+        return (isinstance(other, self.__class__) and
+                self.package_name == other.package_name)
+
+
+def parse_data(raw_data, include_comments=True):
+    # first elem is None to simplify checks of last elem in requirements
+    requirements = [None]
+    for line in raw_data.split("\n"):
+        if line.startswith("#"):
+            if not include_comments:
+                continue
+
+            if (isinstance(requirements[-1], Comment) and
+                    UPPER_LIMIT_TAG in line):
+                requirements[-1].is_finished = True
+
+            if getattr(requirements[-1], "is_finished", True):
+                requirements.append(Comment())
+
+            requirements[-1].append(line)
+        else:
+            if (isinstance(requirements[-1], Comment) and
+                    not requirements[-1].is_finished):
+                requirements[-1].finish_him()
+            if line == "":
+                requirements.append("")
+            else:
+                # parse_line
+                req = Requirement.parse_line(line)
+                if req:
+                    if (isinstance(requirements[-1], Comment) and
+                            str(requirements[-1]).endswith(UPPER_LIMIT_TAG)):
+                        req.upper_limit_flag = True
+                    requirements.append(req)
+    if not requirements[-1]:
+        requirements.pop()
+    return requirements[1:]
+
+
+def _read_requirements():
+    """Read all rally requirements."""
+    LOG.info("Reading rally requirements...")
+    for file_name in RALLY_REQUIREMENTS_FILES:
+        LOG.debug("Try to read '%s'." % file_name)
+        with open(file_name) as f:
+            data = f.read()
+        LOG.info("Parsing requirements from %s." % file_name)
+        yield file_name, parse_data(data)
+
+
+def _write_requirements(filename, requirements):
+    """Saves requirements to file."""
+    LOG.info("Saving requirements to %s." % filename)
+    with open(filename, "w") as f:
+        for entity in requirements:
+            f.write(str(entity))
+            f.write("\n")
+
+
+def _sync():
+    LOG.info("Obtaining global-requirements...")
+    for i in range(0, len(GLOBAL_REQUIREMENTS_LOCATIONS)):
+        url = GLOBAL_REQUIREMENTS_LOCATIONS[i] + GLOBAL_REQUIREMENTS_FILENAME
+        LOG.debug("Try to obtain global-requirements from %s" % url)
+        try:
+            raw_gr = requests.get(url).text
+        except requests.ConnectionError as e:
+            LOG.exception(e)
+            if i == len(GLOBAL_REQUIREMENTS_LOCATIONS) - 1:
+                # there are no more urls to try
+                raise Exception("Unable to obtain %s" %
+                                GLOBAL_REQUIREMENTS_FILENAME)
+        else:
+            break
+
+    LOG.info("Parsing global-requirements...")
+    # NOTE(andreykurilin): global-requirements includes comments which can be
+    #   unrelated to Rally project.
+    gr = parse_data(raw_gr, include_comments=False)
+    for filename, requirements in _read_requirements():
+        for i in range(0, len(requirements)):
+            if isinstance(requirements[i], Requirement):
+                try:
+                    gr_item = gr[gr.index(requirements[i])]
+                except ValueError:
+                    # it not g-r requirements
+                    pass
+                else:
+                    requirements[i].version = gr_item.version
+        yield filename, requirements
+
+
+def sync():
+    """Synchronizes Rally requirements with OpenStack global-requirements."""
+    for filename, requirements in _sync():
+        _write_requirements(filename, requirements)
+
+
+def format_requirements():
+    """Obtain package licenses from pypy and write requirements to file."""
+    for filename, requirements in _read_requirements():
+        _write_requirements(filename, requirements)
+
+
+def add_uppers():
+    """Obtains latest version of packages and put them to requirements."""
+    for filename, requirements in _sync():
+        LOG.info("Obtaining latest versions of packages from %s." % filename)
+        for req in requirements:
+            if isinstance(req, Requirement):
+                req.sync_max_version_with_pypy()
+        _write_requirements(filename, requirements)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="Python Requirement Manager for Rally",
+        description=__doc__.strip(),
+        add_help=True
+    )
+
+    action_groups = parser.add_mutually_exclusive_group()
+    action_groups.add_argument("--format",
+                               action="store_const",
+                               const=format_requirements,
+                               dest="action")
+    action_groups.add_argument("--add-upper",
+                               action="store_const",
+                               const=add_uppers,
+                               dest="action")
+    action_groups.set_defaults(action=sync)
+    parser.parse_args(sys.argv[1:]).action()
+
+if __name__ == "__main__":
+    main()

tests/hacking/checks.py

@@ -154,7 +154,8 @@ def check_import_of_logging(logical_line, physical_line, filename):
 
     excluded_files = ["./rally/common/logging.py",
                       "./tests/unit/test_logging.py",
-                      "./tests/ci/rally_verify.py"]
+                      "./tests/ci/rally_verify.py",
+                      "./tests/ci/sync_requirements.py"]
 
     forbidden_imports = ["from oslo_log",
                          "import oslo_log",

tox.ini

@@ -63,6 +63,10 @@ commands =
 commands =
     oslo-config-generator --config-file etc/rally/rally-config-generator.conf
 
+[testenv:requirements]
+deps = requests[security]
+commands = python {toxinidir}/tests/ci/sync_requirements.py {posargs}
+
 [flake8]
 ignore = H703
 show-source = true