From 43da28e867e91981fb0988c76f64bb3e526fbd06 Mon Sep 17 00:00:00 2001 From: Andrey Kurilin Date: Wed, 7 Sep 2016 16:26:17 +0300 Subject: [PATCH] [requirements] Update requirements * delete upper limits * upgrades sync_requirements file: - write constant values of versions in case of min == max versions - ignore update version in case of [do-not-touch] tag Change-Id: I8b28761b6c365eb3ecbaabf4142edacf36a1a213 --- requirements.txt | 76 +++++++++++++-------------- test-requirements.txt | 25 +++++---- tests/ci/sync_requirements.py | 97 +++++++++++++++++++++-------------- 3 files changed, 109 insertions(+), 89 deletions(-) diff --git a/requirements.txt b/requirements.txt index d45c5478..f4caae84 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,45 +1,41 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -alembic>=0.8.4,<=0.8.7 # MIT -boto>=2.32.1,<=2.42.0 # MIT -decorator>=3.4.0,<=4.0.10 # new BSD License -Jinja2>=2.8,<=2.8 # BSD -# [constant-upper-limit] -jsonschema!=2.5.0,>=2.0.0,<3.0.0 # MIT -netaddr!=0.7.16,>=0.7.12,<=0.7.18 # BSD -oslo.config>=3.14.0,<=3.17.0 # Apache Software License -oslo.db>=4.10.0,<=4.13.1 # Apache Software License -oslo.i18n>=2.1.0,<=3.9.0 # Apache Software License -oslo.log>=1.14.0,<=3.16.0 # Apache Software License -oslo.serialization>=1.10.0,<=2.13.0 # Apache Software License -oslo.utils>=3.16.0,<=3.16.0 # Apache Software License -paramiko>=2.0,<=2.0.2 # LGPL -pbr>=1.6,<=1.10.0 # Apache Software License -# [constant-upper-limit] +alembic>=0.8.4 # MIT +boto>=2.32.1 # MIT +decorator>=3.4.0 # new BSD License +Jinja2>=2.8 # BSD +jsonschema>=2.0.0,!=2.5.0,<3.0.0 # MIT +netaddr>=0.7.12,!=0.7.16 # BSD +oslo.config>=3.14.0 # Apache Software License +oslo.db>=4.10.0 # Apache Software License +oslo.i18n>=2.1.0 # Apache Software License +oslo.log>=1.14.0 # Apache Software License +oslo.serialization>=1.10.0 # Apache Software License +oslo.utils>=3.16.0 # Apache Software License +paramiko>=2.0 # LGPL +pbr>=1.6 # Apache Software License PrettyTable>=0.7,<0.8 # BSD -PyYAML>=3.1.0,<=3.12 # MIT -python-designateclient>=1.5.0,<=2.3.0 # Apache License, Version 2.0 -python-glanceclient!=2.4.0,>=2.3.0,<=2.5.0 # Apache License, Version 2.0 -python-keystoneclient!=2.1.0,>=2.0.0,<=3.5.0 # Apache Software License -keystoneauth1>=2.10.0,<=2.12.1 # Apache Software License -python-novaclient!=2.33.0,>=2.29.0,<=6.0.0 # Apache License, Version 2.0 -python-neutronclient>=5.1.0,<=6.0.0 # Apache Software License -python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0,<=1.9.0 # Apache Software License -python-manilaclient>=1.10.0,<=1.11.0 # Apache Software License -python-monascaclient>=1.1.0,<=1.2.0 # Apache Software License -python-heatclient>=1.4.0,<=1.4.0 # Apache Software License -python-ceilometerclient>=2.5.0,<=2.6.0 # Apache Software License -python-ironicclient>=1.6.0,<=1.7.0 # Apache Software License -python-saharaclient>=0.18.0,<=0.18.0 # Apache License, Version 2.0 -python-troveclient>=2.2.0,<=2.5.0 # Apache Software License -python-zaqarclient>=1.0.0,<=1.2.0 # Apache Software License -python-swiftclient>=2.2.0,<=3.1.0 # Apache Software License -python-watcherclient>=0.23.0,<=0.25.0 # Apache Software License -python-subunit>=0.0.18,<=1.2.0 -requests>=2.10.0,<=2.11.1 # Apache License, Version 2.0 -# [constant-upper-limit] +PyYAML>=3.1.0 # MIT +python-designateclient>=1.5.0 # Apache License, Version 2.0 +python-glanceclient>=2.3.0,!=2.4.0 # Apache License, Version 2.0 +python-keystoneclient>=2.0.0,!=2.1.0 # Apache Software License +keystoneauth1>=2.10.0 # Apache Software License +python-novaclient>=2.29.0,!=2.33.0 # Apache License, Version 2.0 +python-neutronclient>=5.1.0 # Apache Software License +python-cinderclient>=1.6.0,!=1.7.0,!=1.7.1 # Apache Software License +python-manilaclient>=1.10.0 # Apache Software License +python-monascaclient>=1.1.0 # Apache Software License +python-heatclient>=1.4.0 # Apache Software License +python-ceilometerclient>=2.5.0 # Apache Software License +python-ironicclient>=1.6.0 # Apache Software License +python-saharaclient>=0.18.0 # Apache License, Version 2.0 +python-troveclient>=2.2.0 # Apache Software License +python-zaqarclient>=1.0.0 # Apache Software License +python-swiftclient>=2.2.0 # Apache Software License +python-watcherclient>=0.23.0 # Apache Software License +python-subunit>=0.0.18 +requests>=2.10.0 # Apache License, Version 2.0 SQLAlchemy>=1.0.10,<1.1.0 # MIT -# [constant-upper-limit] -sphinx!=1.3b1,>=1.2.1,<1.3 # BSD -six>=1.9.0,<=1.10.0 # MIT +sphinx>=1.2.1,!=1.3b1,<1.3 # BSD +six>=1.9.0 # MIT diff --git a/test-requirements.txt b/test-requirements.txt index 0a15ae2d..b9387f2b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,18 +1,21 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -# [constant-upper-limit] + +# [do-not-touch] we need to modify our code to support latest release of this +# lib hacking>=0.9.2,<0.10 # Apache Software License -pytest>=2.7,<=3.0.2 # MIT -pytest-cov>=2.2.1,<=2.3.1 # MIT -pytest-html>=1.10.0,<=1.10.0 # Mozilla Public License 2.0 (MPL 2.0) + +pytest>=2.7 # MIT +pytest-cov>=2.2.1 # MIT +pytest-html>=1.10.0 # Mozilla Public License 2.0 (MPL 2.0) pytest-xdist # MIT -coverage>=3.6,<=4.2 # Apache License, Version 2.0 -ddt>=1.0.1,<=1.1.0 -mock>=2.0,<=2.0.0 -python-dateutil>=2.4.2,<=2.5.3 # Simplified BSD -testtools>=1.4.0,<=2.2.0 +coverage>=3.6 # Apache License, Version 2.0 +ddt>=1.0.1 +mock>=2.0 +python-dateutil>=2.4.2 # Simplified BSD +testtools>=1.4.0 -oslosphinx!=3.4.0,>=2.5.0,<=4.7.0 # Apache Software License -oslotest>=1.10.0,<=2.10.0 # Apache Software License +oslosphinx>=2.5.0,!=3.4.0 # Apache Software License +oslotest>=1.10.0 # Apache Software License diff --git a/tests/ci/sync_requirements.py b/tests/ci/sync_requirements.py index 23c158c2..5b01e523 100644 --- a/tests/ci/sync_requirements.py +++ b/tests/ci/sync_requirements.py @@ -43,13 +43,13 @@ RALLY_REQUIREMENTS_FILES = ( "test-requirements.txt", "optional-requirements.txt" ) -UPPER_LIMIT_TAG = "[constant-upper-limit]" +DO_NOT_TOUCH_TAG = "[do-not-touch]" class Comment(object): - def __init__(self, s=None): + def __init__(self, s=None, finished=False): self._comments = [] - self.is_finished = False + self.is_finished = finished if s: self.append(s) @@ -82,7 +82,7 @@ class Requirement(object): self.version = version self._license = None self._pypy_info = None - self.upper_limit_flag = False + self.do_not_touch = False def sync_max_version_with_pypy(self): if isinstance(self.version, dict) and not self.upper_limit_flag: @@ -151,26 +151,41 @@ class Requirement(object): def __str__(self): if isinstance(self.version, dict): - version = "" - if self.version["ne"]: - version += ",".join(self.version["ne"]) - if self.version["min"]: - if version: - version += "," - version += self.version["min"] - if self.version["max"]: - if version: - version += "," - version += self.version["max"] + version = [] + + min_equal_to_max = False + if self.version["min"] and self.version["max"]: + if (self.version["min"].startswith(">=") and + self.version["max"].startswith("<=") and + self.version["min"][2:] == self.version["max"][2:]): + # min and max versions are equal there is no need to write + # both of them + min_equal_to_max = True + version.append("==%s" % self.version["min"][2:]) + + if not min_equal_to_max and self.version["min"]: + version.append(self.version["min"]) + + if not min_equal_to_max and self.version["ne"]: + version.extend(self.version["ne"]) + + if not min_equal_to_max and self.version["max"]: + version.append(self.version["max"]) + + version = ",".join(version) else: - version = self.version + if self.do_not_touch: + version = self.version + else: + # remove const version + version = ">=%s" % self.version[2:] string = "%s%s" % (self.package_name, version) if self.license: # NOTE(andreykurilin): When I start implementation of this script, - # python-keystoneclient dependency took around ~45-55, so let's - # use this length as indent. Feel free to modify it to lower or - # greater value. + # python-keystoneclient dependency string took around ~45-55 + # chars, so let's use this length as indent. Feel free to modify + # it to lower or greater value. magic_number = 55 if len(string) < magic_number: indent = magic_number - len(string) @@ -192,30 +207,33 @@ def parse_data(raw_data, include_comments=True): if not include_comments: continue - if (isinstance(requirements[-1], Comment) and - UPPER_LIMIT_TAG in line): - requirements[-1].is_finished = True - if getattr(requirements[-1], "is_finished", True): requirements.append(Comment()) requirements[-1].append(line) + elif line == "": + # just empty line + if isinstance(requirements[-1], Comment): + requirements[-1].finish_him() + requirements.append(Comment(finished=True)) else: if (isinstance(requirements[-1], Comment) and not requirements[-1].is_finished): requirements[-1].finish_him() - if line == "": - requirements.append("") - else: - # parse_line - req = Requirement.parse_line(line) - if req: - if (isinstance(requirements[-1], Comment) and - str(requirements[-1]).endswith(UPPER_LIMIT_TAG)): - req.upper_limit_flag = True - requirements.append(req) - if not requirements[-1]: - requirements.pop() + # parse_line + req = Requirement.parse_line(line) + if req: + if (isinstance(requirements[-1], Comment) and + DO_NOT_TOUCH_TAG in str(requirements[-1])): + req.do_not_touch = True + requirements.append(req) + for i in range(len(requirements) - 1, 0, -1): + # remove empty lines at the end of file + if isinstance(requirements[i], Comment): + if str(requirements[i]) == "": + requirements.pop(i) + else: + break return requirements[1:] @@ -261,12 +279,14 @@ def _sync(): gr = parse_data(raw_gr, include_comments=False) for filename, requirements in _read_requirements(): for i in range(0, len(requirements)): - if isinstance(requirements[i], Requirement): + if (isinstance(requirements[i], Requirement) and + not requirements[i].do_not_touch): try: gr_item = gr[gr.index(requirements[i])] except ValueError: # it not g-r requirements - pass + if isinstance(requirements[i].version, dict): + requirements[i].version["max"] = None else: requirements[i].version = gr_item.version yield filename, requirements @@ -290,7 +310,8 @@ def add_uppers(): LOG.info("Obtaining latest versions of packages from %s." % filename) for req in requirements: if isinstance(req, Requirement): - req.sync_max_version_with_pypy() + if isinstance(req.version, dict) and not req.version["max"]: + req.sync_max_version_with_pypy() _write_requirements(filename, requirements)