one global-requirements.txt to rule them all

... and in the darkness bind them.

This moves the contents of requirements.txt and
test-requirements.txt to a single file, global-requirements.txt.
This means that regardless of whether a requirement is in
either file it will get checked. This massively simplifies the
checking and upgrading of req and test-req files.

This also adds in a unit testing framework for the update.py
script to ensure it does what we think it does. This is
accomplished by making a fake tree with a set of req and test-req
files, running update.py, and ensuring the updates were made
that we expected. It includes testing for the oslo url case,
as well as test-requires.

Change-Id: Ib9b86ade4cb8317509e218aec31f32e5d08f4035

.testr.conf

@@ -0,0 +1,8 @@
+[DEFAULT]
+test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
+             OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
+             OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
+             ${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION
+
+test_id_option=--load-list $IDFILE
+test_list_option=--list

global-requirements.txt

@@ -81,3 +81,50 @@ websockify>=0.5.1,<0.6
 wsgiref>=0.1.2
 WSME>=0.5b2
 xattr>=0.4
+# Testing tools below, which are typically in test-requires.txt
+
+cliff-tablib>=1.0
+configobj
+coverage>=3.6
+discover
+django-nose
+docutils==0.9.1
+feedparser
+fixtures>=0.3.12
+flake8==2.0
+hacking>=0.5.6,<0.7
+hgtools # dependency of pytest-runner that is only in setup_requires
+hp3parclient>=1.0.0
+httpretty>=0.6.3
+keyring
+mock>=0.8.0
+mox>=0.5.3
+MySQL-python
+nose
+nose-exclude
+nosehtmloutput>=0.0.3
+nosexcover
+openstack.nose_plugin>=0.7
+pep8==1.4.5
+psycopg2
+pyflakes==0.7.2
+pylint==0.25.2
+pysendfile==2.0.0
+pysqlite
+python-ldap==2.3.13
+python-subunit
+pytest-runner # dependency of keyring that is only in setup_requires
+pyzmq==2.2.0.1
+redis
+selenium
+sphinx>=1.1.2
+sphinxcontrib-httpdomain
+sphinxcontrib-pecanwsme>=0.2
+oslo.sphinx
+swift
+testrepository>=0.0.17
+testresources<0.3
+testscenarios>=0.4,<0.5
+testtools>=0.9.32
+unittest2
+WebTest==1.3.3

test-requirements.txt

@@ -1,45 +1,3 @@
-cliff-tablib>=1.0
-configobj
-coverage>=3.6
-discover
-django-nose
-docutils==0.9.1
-feedparser
-fixtures>=0.3.12
-flake8==2.0
-hacking>=0.5.6,<0.7
-hgtools # dependency of pytest-runner that is only in setup_requires
-hp3parclient>=1.0.0
-httpretty>=0.6.3
-keyring
-mock>=0.8.0
-mox>=0.5.3
-MySQL-python
-nose
-nose-exclude
-nosehtmloutput>=0.0.3
-nosexcover
-openstack.nose_plugin>=0.7
-pep8==1.4.5
-psycopg2
-pyflakes==0.7.2
-pylint==0.25.2
-pysendfile==2.0.0
-pysqlite
-python-ldap==2.3.13
-python-subunit
-pytest-runner # dependency of keyring that is only in setup_requires
-pyzmq==2.2.0.1
-redis
-selenium
-sphinx>=1.1.2
-sphinxcontrib-httpdomain
-sphinxcontrib-pecanwsme>=0.2
-oslo.sphinx
-swift
 testrepository>=0.0.17
-testresources<0.3
 testscenarios>=0.4,<0.5
 testtools>=0.9.32
-unittest2
-WebTest==1.3.3

tests/files/gr-base.txt

@@ -0,0 +1,131 @@
+alembic>=0.4.1
+amqplib>=0.6.1
+anyjson>=0.3.3
+argparse
+Babel>=0.9.6
+boto>=2.4.0
+cffi
+Cheetah>=2.4.4
+cliff>=1.4
+d2to1>=0.2.10,<0.3
+Django>=1.4,<1.6
+django_compressor>=1.3
+django_openstack_auth>=1.1.0
+dnspython>=1.9.4
+eventlet>=0.12.0
+extras
+Flask==0.9
+greenlet>=0.3.2
+happybase>=0.4
+httplib2
+iso8601>=0.1.4
+Jinja2
+jsonrpclib
+jsonschema>=1.0.0,!=1.4.0,<2
+kazoo>=0.9,<=1.1
+lesscpy>=0.9h
+kombu>=2.4.8
+lockfile>=0.8
+lxml>=2.3
+msgpack-python
+netaddr
+netifaces>=0.5
+oauth2
+oslo.config>=1.1.0
+pam>=0.1.4
+paramiko>=1.8.0
+passlib
+Paste
+PasteDeploy>=1.5.0
+pbr>=0.5.16,<0.6
+pecan>=0.2.0
+pip>=1.0
+PrettyTable>=0.6,<0.8
+psutil<1.0
+pyasn1
+pycrypto>=2.6
+pymongo>=2.4
+pyOpenSSL
+pyparsing>=1.5.7,<2.0
+# OpenStack clients. None of these should have an upper bound
+# as that has implications for testing in the gate. An exception
+# is currently being made for neutron client because of the need
+# for an incompatible change in their next release.
+python-cinderclient>=1.0.4
+python-ceilometerclient>=1.0.2
+python-heatclient>=0.2.3
+python-glanceclient>=0.9.0
+python-keystoneclient>=0.3.0
+python-memcached
+python-neutronclient>=2.2.3,<3
+python-novaclient>=2.12.0
+python-quantumclient>=2.2.0
+python-swiftclient>=1.2
+python-troveclient
+pytz>=2010h
+pyudev
+PyYAML>=3.1.0
+qpid-python
+requests>=1.1,<1.2.3
+Routes>=1.12.3
+setuptools_git>=0.4
+simplejson>=2.0.9
+six
+sockjs-tornado>=1.0.0,<2.0.0
+SQLAlchemy>=0.7,<=0.7.99
+sqlalchemy-migrate>=0.7
+stevedore>=0.10
+suds>=0.4
+warlock>=0.7.0,<2
+WebOb>=1.2.3,<1.3
+websockify>=0.5.1,<0.6
+wsgiref>=0.1.2
+WSME>=0.5b2
+xattr>=0.4
+# Testing tools below, which are typically in test-requires.txt
+
+cliff-tablib>=1.0
+configobj
+coverage>=3.6
+discover
+django-nose
+docutils==0.9.1
+feedparser
+fixtures>=0.3.12
+flake8==2.0
+hacking>=0.5.6,<0.7
+hgtools # dependency of pytest-runner that is only in setup_requires
+hp3parclient>=1.0.0
+httpretty>=0.6.3
+keyring
+mock>=0.8.0
+mox>=0.5.3
+MySQL-python
+nose
+nose-exclude
+nosehtmloutput>=0.0.3
+nosexcover
+openstack.nose_plugin>=0.7
+pep8==1.4.5
+psycopg2
+pyflakes==0.7.2
+pylint==0.25.2
+pysendfile==2.0.0
+pysqlite
+python-ldap==2.3.13
+python-subunit
+pytest-runner # dependency of keyring that is only in setup_requires
+pyzmq==2.2.0.1
+redis
+selenium
+sphinx>=1.1.2
+sphinxcontrib-httpdomain
+sphinxcontrib-pecanwsme>=0.2
+oslo.sphinx
+swift
+testrepository>=0.0.17
+testresources<0.3
+testscenarios>=0.4,<0.5
+testtools>=0.9.32
+unittest2
+WebTest==1.3.3

tests/files/project-with-oslo-tar.txt

@@ -0,0 +1,35 @@
+d2to1>=0.2.10,<0.3
+pbr>=0.5.16,<0.6
+SQLAlchemy>=0.7.8,<0.7.99
+Cheetah>=2.4.4
+amqplib>=0.6.1
+anyjson>=0.2.4
+argparse
+boto
+eventlet>=0.9.17
+kombu>=1.0.4
+lxml>=2.3
+routes>=1.12.3
+WebOb==1.2.3
+greenlet>=0.3.1
+PasteDeploy>=1.5.0
+paste
+sqlalchemy-migrate>=0.7.2
+netaddr>=0.7.6
+suds>=0.4
+paramiko
+pyasn1
+Babel>=0.9.6
+iso8601>=0.1.4
+requests>=1.1,<1.2.1 # order-dependent python-cinderclient req cap, bug 1182271
+python-cinderclient>=1.0.1
+python-neutronclient>=2.2.3,<3.0.0
+python-glanceclient>=0.9.0
+python-keystoneclient>=0.2.0
+six
+stevedore>=0.10
+websockify<0.4
+pyparsing>=1.5.7,<2.0 # order-dependent python-quantumclient req, bug 1191866
+
+-f http://tarballs.openstack.org/oslo.config/oslo.config-1.2.0a3.tar.gz#egg=oslo.config-1.2.0a3
+oslo.config>=1.2.0a3

tests/files/project.txt

@@ -0,0 +1,43 @@
+# The greenlet package must be compiled with gcc and needs
+# the Python.h headers. Make sure you install the python-dev
+# package to get the right headers...
+greenlet>=0.3.1
+
+# < 0.8.0/0.8 does not work, see https://bugs.launchpad.net/bugs/1153983
+SQLAlchemy>=0.7.8,<=0.7.99
+anyjson>=0.3.3
+eventlet>=0.9.12
+PasteDeploy
+routes
+WebOb>=1.2
+wsgiref
+argparse
+boto
+sqlalchemy-migrate>=0.7
+httplib2
+kombu>2.4.7
+pycrypto>=2.1.0alpha1
+iso8601>=0.1.4
+oslo.config>=1.1.0
+
+
+# For Swift storage backend.
+python-swiftclient>=1.2,<2
+
+# Note you will need gcc buildtools installed and must
+# have installed libxml headers for lxml to be successfully
+# installed using pip, therefore you will need to install the
+# libxml2-dev and libxslt-dev Ubuntu packages.
+lxml
+
+# For paste.util.template used in keystone.common.template
+Paste
+
+passlib
+jsonschema
+python-cinderclient>=1.0.4
+python-keystoneclient>=0.2.0
+pyOpenSSL
+
+# Required by openstack.common libraries
+six

tests/files/test-project.txt

@@ -0,0 +1,18 @@
+hacking>=0.5.6,<0.7
+coverage>=3.6
+discover
+feedparser
+fixtures>=0.3.12
+mox==0.5.3
+MySQL-python
+psycopg2
+pylint==0.25.2
+# Imported by ldapdns so required to generate
+# the sample configuration file
+python-ldap==2.3.13
+python-subunit
+setuptools_git>=0.4
+sphinx>=1.1.2
+oslo.sphinx
+testrepository>=0.0.13
+testtools>=0.9.27

tests/test_update.py

@@ -0,0 +1,83 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import os.path
+import shutil
+import subprocess
+import sys
+import tempfile
+import testtools
+
+
+def _load_req(fname):
+    with open(fname) as f:
+        content = map(lambda x: x.rstrip(), f.readlines())
+        print content
+        return content
+
+
+class UpdateTest(testtools.TestCase):
+
+    def setUp(self):
+        super(UpdateTest, self).setUp()
+        self.dir = tempfile.mkdtemp()
+        self.project_dir = os.path.join(self.dir, "project")
+        self.oslo_dir = os.path.join(self.dir, "project_with_oslo")
+
+        self.req_file = os.path.join(self.dir, "global-requirements.txt")
+        self.proj_file = os.path.join(self.project_dir, "requirements.txt")
+        self.oslo_file = os.path.join(self.oslo_dir, "requirements.txt")
+        self.proj_test_file = os.path.join(self.project_dir,
+                                           "test-requirements.txt")
+        os.mkdir(self.project_dir)
+        os.mkdir(self.oslo_dir)
+
+        shutil.copy("tests/files/gr-base.txt", self.req_file)
+        shutil.copy("tests/files/project-with-oslo-tar.txt", self.oslo_file)
+        shutil.copy("tests/files/project.txt", self.proj_file)
+        shutil.copy("tests/files/test-project.txt", self.proj_test_file)
+        shutil.copy("update.py", os.path.join(self.dir, "update.py"))
+
+        # now go call update and see what happens
+        os.chdir(self.dir)
+        subprocess.call([sys.executable, "update.py", "project"])
+        subprocess.call([sys.executable, "update.py", "project_with_oslo"])
+
+    def test_requirements(self):
+        reqs = _load_req(self.req_file)
+        self.assertIn("jsonschema>=1.0.0,!=1.4.0,<2", reqs)
+
+    def test_project(self):
+        reqs = _load_req(self.proj_file)
+        # ensure various updates take
+        self.assertIn("jsonschema>=1.0.0,!=1.4.0,<2", reqs)
+        self.assertIn("python-keystoneclient>=0.3.0", reqs)
+        self.assertIn("SQLAlchemy>=0.7,<=0.7.99", reqs)
+
+    def test_project_with_oslo(self):
+        reqs = _load_req(self.oslo_file)
+        oslo_tar = ("-f http://tarballs.openstack.org/oslo.config/"
+                    "oslo.config-1.2.0a3.tar.gz#egg=oslo.config-1.2.0a3")
+        self.assertIn(oslo_tar, reqs)
+        self.assertIn("oslo.config>=1.1.0", reqs)
+
+    def test_test_project(self):
+        reqs = _load_req(self.proj_test_file)
+        self.assertIn("testtools>=0.9.32", reqs)
+        self.assertIn("testrepository>=0.0.17", reqs)
+        # make sure we didn't add something we shouldn't
+        self.assertNotIn("sphinxcontrib-pecanwsme>=0.2", reqs)

tox.ini

@@ -0,0 +1,12 @@
+[tox]
+envlist = py27
+
+[testenv]
+setenv = VIRTUAL_ENV={envdir}
+         LANG=en_US.UTF-8
+         LANGUAGE=en_US:en
+         LC_ALL=C
+deps = -U
+      -r{toxinidir}/test-requirements.txt
+commands =
+  python setup.py testr --slowest --testr-args='{posargs}'

update.py

@@ -66,30 +66,12 @@ def _parse_reqs(filename):
     return reqs
 
 
-def _copy_requires(source_path, dest_dir):
-    """Copy requirements files."""
-
-    target_map = {
-        'requirements.txt': ('requirements.txt', 'tools/pip-requires'),
-        'test-requirements.txt': (
-            'test-requirements.txt', 'tools/test-requires'),
-    }
-    for dest in target_map[source_path]:
-        dest_path = os.path.join(dest_dir, dest)
-        if os.path.exists(dest_path):
-            break
-
-    # Catch the fall through
-    if not os.path.exists(dest_path):
-        # This can happen, we try all paths
-        return
-
-    source_reqs = _parse_reqs(source_path)
-
+def _sync_requirements_file(source_reqs, dest_path):
+    dest_reqs = []
     with open(dest_path, 'r') as dest_reqs_file:
         dest_reqs = dest_reqs_file.readlines()
 
-    print "Syncing %s" % source_path
+    print "Syncing %s" % dest_path
 
     with open(dest_path, 'w') as new_reqs:
         for old_line in dest_reqs:
@@ -111,9 +93,24 @@ def _copy_requires(source_path, dest_dir):
                 new_reqs.write("%s\n" % source_reqs[old_pip])
 
 
+def _copy_requires(source_path, dest_dir):
+    """Copy requirements files."""
+
+    source_reqs = _parse_reqs(source_path)
+
+    target_files = (
+        'requirements.txt', 'tools/pip-requires',
+        'test-requirements.txt', 'tools/test-requires')
+
+    for dest in target_files:
+        dest_path = os.path.join(dest_dir, dest)
+        if os.path.exists(dest_path):
+            print "_sync_requirements_file(%s, %s)" % (source_reqs, dest_path)
+            _sync_requirements_file(source_reqs, dest_path)
+
+
 def main(argv):
-    for req in ('requirements.txt', 'test-requirements.txt'):
-        _copy_requires(req, argv[0])
+    _copy_requires('global-requirements.txt', argv[0])
 
 
 if __name__ == "__main__":