Create integration tests requirements on the fly

The problem of integration tests was that they use their own
requirements.txt that is unable to auto-sync with global requirements.

This patch removes dedicated requirements.txt and replaces it with a
stub file listing names (and possibly versions) of the packages needed.
Then a special script parses this stub file, and for every package
that is present in main project requirements files it pulls the versions
from main project requirements, generating and installing requirements
for the integration tests on the fly.

This will help keeping requirements for the integration tests always in
sync with main project requirements.

Change-Id: Ie79338cc10cc101fbf15b51c7923e3a7b8e4fbb4
Closes-Bug: #1490866

.gitignore

@@ -21,6 +21,8 @@ cover
 .pydevproject
 doc/source/api/
 etc/heat/heat.conf.sample
+# integration tests requirements are auto-generated from stub file
+heat_integrationtests/requirements.txt
 
 # Files created by releasenotes build
-releasenotes/build
+releasenotes/build

heat_integrationtests/install-requirements

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Generate and install requirements from stub file and source files."""
+
+import argparse
+import subprocess
+import sys
+
+import pip
+
+parser = argparse.ArgumentParser(description='Sync requirements.')
+parser.add_argument('--stub', metavar='STUBFILE',
+                    required=True,
+                    help="File with requirements stubs.")
+parser.add_argument('--source', metavar='SOURCE',
+                    required=True, action='append',
+                    help="Source file to sync requirements from. "
+                    "May be supplied several times.")
+parser.add_argument('-t', '--target', metavar='TARGET',
+                    required=True,
+                    help="Target file to write synced requirements to.")
+parser.add_argument('pipopts', metavar='PIP OPTIONS',
+                    nargs=argparse.REMAINDER,
+                    help='Options to pass to "pip install".')
+
+args = parser.parse_args()
+
+sources = {}
+for requirements_file in args.source:
+    rqs = pip.req.req_file.parse_requirements(requirements_file,
+                                              session=False)
+    sources.update({s.name: s for s in rqs})
+stubs = list(pip.req.req_file.parse_requirements(args.stub,
+                                                 session=False))
+reqs = []
+for r in stubs:
+    if r.name in sources:
+        # safe-guard for future additions to stub file
+        if r.specifier:
+            sys.exit("ERROR: package '%(pkg)s' in stub file %(stub)s "
+                     "has version specified but is also present "
+                     "in provided sources requirements. "
+                     "Please remove version from the stub file." % {
+                         'pkg': r.name, 'stub': args.stub})
+        reqs.append(sources[r.name])
+    else:
+        reqs.append(r)
+
+with open(args.target, 'w') as target:
+    target.write('\n'.join([str(r.req) for r in reqs]))
+
+pip_install = ['pip', 'install', '-r', args.target]
+pip_install.extend(args.pipopts)
+
+sys.exit(subprocess.call(pip_install))

heat_integrationtests/requirements.stub

@@ -0,0 +1,31 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+# These are stubs for requirements. This file must be processed with
+# heat_integrationtests/install-requirements to generate and install the real
+# requirements.txt where versions for packages present in
+# main project requirements will be synced from main requirements files.
+
+pbr
+kombu
+os-collect-config
+oslo.log
+oslo.messaging
+oslo.concurrency
+oslo.config
+oslo.utils
+paramiko
+python-ceilometerclient
+python-cinderclient
+python-keystoneclient
+python-heatclient
+python-neutronclient
+python-novaclient
+python-swiftclient
+PyYAML
+requests
+six
+testrepository
+testscenarios
+testtools

heat_integrationtests/requirements.txt

@@ -1,25 +0,0 @@
-# The order of packages is significant, because pip processes them in the order
-# of appearance. Changing the order has an impact on the overall integration
-# process, which may cause wedges in the gate later.
-pbr<2.0,>=0.11
-kombu>=3.0.7
-os-collect-config # Apache-2.0
-oslo.log>=1.2.0 # Apache-2.0
-oslo.messaging!=1.12.0,>=1.8.0 # Apache-2.0
-oslo.concurrency>=2.1.0
-oslo.config>=1.11.0 # Apache-2.0
-oslo.utils>=1.6.0 # Apache-2.0
-paramiko>=1.13.0
-python-ceilometerclient>=1.5.0
-python-cinderclient>=1.3.1
-python-keystoneclient>=1.6.0
-python-heatclient>=0.6.0
-python-neutronclient>=2.6.0
-python-novaclient>=2.29.0
-python-swiftclient>=2.2.0
-PyYAML>=3.1.0
-requests>=2.5.2
-six>=1.9.0
-testrepository>=0.0.18
-testscenarios>=0.4
-testtools>=1.4.0

test-requirements.txt

@@ -12,7 +12,6 @@ mox3>=0.7.0 # Apache-2.0
 PyMySQL>=0.6.2 # MIT License
 oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
 oslotest>=1.10.0 # Apache-2.0
-paramiko>=1.16.0 # LGPL
 qpid-python;python_version=='2.7' # Apache-2.0
 psycopg2>=2.5 # LGPL/ZPL
 sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD
@@ -21,3 +20,6 @@ testscenarios>=0.4 # Apache-2.0/BSD
 testtools>=1.4.0 # MIT
 testresources>=0.2.4 # Apache-2.0/BSD
 reno>=0.1.1 # Apache2
+# Next two are used in integration tests only
+os-collect-config  # Apache-2.0
+paramiko>=1.16.0 # LGPL

tox.ini

@@ -28,7 +28,8 @@ setenv = VIRTUAL_ENV={envdir}
          TESTR_START_DIR=heat_integrationtests
 passenv = OS_*
 usedevelop = False
-deps = -r{toxinidir}/heat_integrationtests/requirements.txt
+install_command = {toxinidir}/heat_integrationtests/install-requirements --stub {toxinidir}/heat_integrationtests/requirements.stub --source {toxinidir}/requirements.txt --source {toxinidir}/test-requirements.txt {packages} {opts}
+deps = -t{toxinidir}/heat_integrationtests/requirements.txt
 commands =
   bash tools/pretty_tox.sh '{posargs}'