Update requirements

* rtd can use only one requirements file
* remove redundant requirements from tests-requirements.txt
* update upper-constraints

Change-Id: I9fa049abbe1fd95b4a3adbc8994225d46571db27

doc/requirements.txt

@@ -1,5 +1,9 @@
-# we are using theme from it
+# main dependencies
-oslosphinx
+-r ../requirements.txt
+sphinx!=1.6.6,!=1.6.7,!=2.1.0                          # BSD
 
-# Rally plugins
+# we are using theme from this package
+oslosphinx                                             # Apache Software License
+
+# plugins
 rally-openstack

requirements.txt

@@ -17,7 +17,7 @@ pbr!=2.1.0                                             # Apache Software License
 PrettyTable<0.8                                        # BSD
 pyOpenSSL                                              # Apache License, Version 2.0
 PyYAML                                                 # MIT
-python-subunit
+python-subunit                                         # Apache-2.0 or BSD
 requests!=2.20.0                                       # Apache License, Version 2.0
 SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8              # MIT
 virtualenv!=16.3.0                                     # MIT

test-requirements.txt

@@ -2,24 +2,20 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-hacking>=3.0,<3.1.0 # Apache-2.0
+hacking>=3.0                                           # Apache Software License
 
-pytest>=2.7                                            # MIT
+pytest                                                 # MIT
 # py.test plugin for measuring coverage.
 pytest-cov>=2.2.1                                      # MIT
 # py.test plugin for generating HTML reports
-pytest-html>=1.10.0                                    # Mozilla Public License 2.0 (MPL 2.0)
+pytest-html                                            # Mozilla Public License 2.0 (MPL 2.0)
 # py.test xdist plugin for distributed testing and loop-on-failing modes
 pytest-xdist                                           # MIT
 
 ddt                                                    # MIT
-python-dateutil                                        # Dual License
+testtools                                              # MIT
-testtools                                              # UNKNOWN
-
-sphinx!=1.6.6,!=1.6.7,!=2.1.0                          # BSD
-oslosphinx                                             # Apache Software License
 
 testresources                                          # UNKNOWN
 
-# needed for functional job
+docutils                                               # public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)
-stestr                                                 # Apache Software License
+Pygments                                               # BSD

tests/ci/sync_requirements.py

@@ -24,6 +24,7 @@ import re
 import sys
 import textwrap
 
+import pkg_resources
 import requests
 
 
@@ -35,7 +36,7 @@ if not LOG.handlers:
 
 GLOBAL_REQUIREMENTS_LOCATIONS = (
     "https://raw.githubusercontent.com/openstack/requirements/master/",
-    "http://opendev.org/openstack/requirements/raw/branch/master/"
+    "http://git.openstack.org/cgit/openstack/requirements/plain/"
 )
 RALLY_REQUIREMENTS_FILES = (
     "requirements.txt",
@@ -256,6 +257,10 @@ class UpperConstraint(PYPIPackage):
     def update(self, version):
         self._version = version
 
+    def fetch_version(self):
+        self._version = None
+        return self.version
+
 
 def parse_data(raw_data, include_comments=True, dependency_cls=Requirement):
     # first elem is None to simplify checks of last elem in requirements
@@ -355,26 +360,21 @@ def update_upper_constraints():
     global_uc = parse_data(raw_g_uc,
                            include_comments=False,
                            dependency_cls=UpperConstraint)
-    with open("upper-constraints.txt") as f:
-        our_uc = parse_data(f.read(),
-                            dependency_cls=UpperConstraint)
-    with open("requirements.txt") as f:
-        our_requirements = parse_data(f.read(), include_comments=False)
 
-    for name, req in our_requirements.items():
+    our_uc = [UpperConstraint(package_name=p.project_name, version=p.version)
-        if isinstance(req, Comment):
+              for p in pkg_resources.working_set
-            continue
+              # do not include the current package at u-c
-        if name not in our_uc:
+              if p.project_name != "rally"]
-            our_uc[name] = UpperConstraint(name)
 
-        if name in global_uc:
+    for package in our_uc:
+        if package.package_name in global_uc:
             # we cannot use whatever we want versions in CI. OpenStack CI
             # ignores versions listed in requirements of
             # particular project and use versions from global u-c file.
             # It means that we need to suggest to use the same versions
-            our_uc[name].update(global_uc[name].version)
+            package.update(global_uc[package.package_name].version)
 
-    our_uc = sorted(our_uc.values(), key=lambda o: o.package_name.upper())
+    our_uc = sorted(our_uc, key=lambda o: o.package_name.upper())
     _write_requirements("upper-constraints.txt", our_uc)
 
 

tox.ini

@@ -50,11 +50,17 @@ commands =
 commands = {posargs}
 
 [testenv:functional]
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+       stestr
 commands =
   find . -type f -name "*.pyc" -delete
   python {toxinidir}/tests/ci/pytest_launcher.py tests/functional --posargs={posargs}
 
 [testenv:functional-py38]
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+       stestr
 basepython = python3.8
 commands =
   find . -type f -name "*.pyc" -delete
@@ -66,8 +72,6 @@ commands = {toxinidir}/tests/ci/cover.sh {posargs}
 [testenv:docs]
 deps =
   -c{toxinidir}/upper-constraints.txt
-  -r{toxinidir}/requirements.txt
-  -r{toxinidir}/test-requirements.txt
   -r{toxinidir}/doc/requirements.txt
 commands =
   rm -rf doc/build

upper-constraints.txt

@@ -1,18 +1,69 @@
-alembic===1.3.0
+alembic===1.4.2
-decorator===4.4.1
+argparse===1.4.0
-Jinja2===2.10.3
+attrs===19.3.0
-jsonschema===3.1.1
+Babel===2.8.0
-oslo.config===6.11.1
+bcrypt===3.1.7
-oslo.db===5.0.2
+certifi===2020.4.5.1
-oslo.log===3.44.1
+cffi===1.14.0
-paramiko===2.6.0
+chardet===3.0.4
-pbr===5.4.3
+cryptography===2.9.2
-PrettyTable===0.7.2
+debtcollector===2.0.1
-pyOpenSSL===19.0.0
+decorator===4.4.2
-PyYAML===5.1.2
+extras===1.0.0
-six===1.13.0
+fixtures===3.0.0
-SQLAlchemy===1.3.13
+idna===2.9
-virtualenv===16.7.7
+importlib-metadata===1.6.0
+iso8601===0.1.12
+Jinja2===2.11.2
+jsonschema===3.2.0
+linecache2===1.0.0
+Mako===1.1.2
+MarkupSafe===1.1.1
+msgpack===0.6.1
+netaddr===0.7.19
+netifaces===0.10.9
+oslo.config===8.0.2
+oslo.context===3.0.2
+oslo.db===8.1.0
+oslo.i18n===4.0.1
+oslo.log===4.1.1
+oslo.serialization===3.1.1
+oslo.utils===4.1.1
+paramiko===2.7.1
+pbr===5.4.5
+pip===19.1.1
+prettytable===0.7.2
+pycparser===2.20
+pyinotify===0.9.6
+PyNaCl===1.3.0
+pyOpenSSL===19.1.0
+pyparsing===2.4.7
+pyrsistent===0.16.0
+python-dateutil===2.8.1
+python-editor===1.0.4
+python-mimeparse===1.6.0
+python-subunit===1.4.0
+pytz===2020.1
+PyYAML===5.3.1
+requests===2.23.0
+rfc3986===1.4.0
+setuptools===41.0.1
+six===1.14.0
+SQLAlchemy===1.3.16
+sqlalchemy-migrate===0.13.0
+sqlparse===0.3.1
+stevedore===1.32.0
+Tempita===0.5.2
+testresources===2.0.1
+testscenarios===0.5.0
+testtools===2.4.0
+traceback2===1.4.0
+unittest2===1.1.0
+urllib3===1.25.9
+virtualenv===20.0.20
+wheel===0.33.4
+wrapt===1.12.1
+zipp===3.1.0
 # see https://github.com/datadriventests/ddt/pull/84 and
 #  https://github.com/datadriventests/ddt/pull/82 for more details
 ddt===1.3.1