From dd50ac63523167b50d5793a0f634a3a049c15c9f Mon Sep 17 00:00:00 2001 From: Pavlo Shchelokovskyy Date: Mon, 31 Aug 2015 14:52:16 +0000 Subject: [PATCH] Create integration tests requirements on the fly The problem of integration tests was that they use their own requirements.txt that is unable to auto-sync with global requirements. This patch removes dedicated requirements.txt and replaces it with a stub file listing names (and possibly versions) of the packages needed. Then a special script parses this stub file, and for every package that is present in main project requirements files it pulls the versions from main project requirements, generating and installing requirements for the integration tests on the fly. This will help keeping requirements for the integration tests always in sync with main project requirements. Change-Id: Ie79338cc10cc101fbf15b51c7923e3a7b8e4fbb4 Closes-Bug: #1490866 --- .gitignore | 4 +- heat_integrationtests/install-requirements | 66 ++++++++++++++++++++++ heat_integrationtests/requirements.stub | 31 ++++++++++ heat_integrationtests/requirements.txt | 25 -------- test-requirements.txt | 4 +- tox.ini | 3 +- 6 files changed, 105 insertions(+), 28 deletions(-) create mode 100755 heat_integrationtests/install-requirements create mode 100644 heat_integrationtests/requirements.stub delete mode 100644 heat_integrationtests/requirements.txt diff --git a/.gitignore b/.gitignore index 27b22138e1..647a561585 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ cover .pydevproject doc/source/api/ etc/heat/heat.conf.sample +# integration tests requirements are auto-generated from stub file +heat_integrationtests/requirements.txt # Files created by releasenotes build -releasenotes/build \ No newline at end of file +releasenotes/build diff --git a/heat_integrationtests/install-requirements b/heat_integrationtests/install-requirements new file mode 100755 index 0000000000..a94d07b3c6 --- /dev/null +++ b/heat_integrationtests/install-requirements @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Generate and install requirements from stub file and source files.""" + +import argparse +import subprocess +import sys + +import pip + +parser = argparse.ArgumentParser(description='Sync requirements.') +parser.add_argument('--stub', metavar='STUBFILE', + required=True, + help="File with requirements stubs.") +parser.add_argument('--source', metavar='SOURCE', + required=True, action='append', + help="Source file to sync requirements from. " + "May be supplied several times.") +parser.add_argument('-t', '--target', metavar='TARGET', + required=True, + help="Target file to write synced requirements to.") +parser.add_argument('pipopts', metavar='PIP OPTIONS', + nargs=argparse.REMAINDER, + help='Options to pass to "pip install".') + +args = parser.parse_args() + +sources = {} +for requirements_file in args.source: + rqs = pip.req.req_file.parse_requirements(requirements_file, + session=False) + sources.update({s.name: s for s in rqs}) +stubs = list(pip.req.req_file.parse_requirements(args.stub, + session=False)) +reqs = [] +for r in stubs: + if r.name in sources: + # safe-guard for future additions to stub file + if r.specifier: + sys.exit("ERROR: package '%(pkg)s' in stub file %(stub)s " + "has version specified but is also present " + "in provided sources requirements. " + "Please remove version from the stub file." % { + 'pkg': r.name, 'stub': args.stub}) + reqs.append(sources[r.name]) + else: + reqs.append(r) + +with open(args.target, 'w') as target: + target.write('\n'.join([str(r.req) for r in reqs])) + +pip_install = ['pip', 'install', '-r', args.target] +pip_install.extend(args.pipopts) + +sys.exit(subprocess.call(pip_install)) diff --git a/heat_integrationtests/requirements.stub b/heat_integrationtests/requirements.stub new file mode 100644 index 0000000000..55a6b27d67 --- /dev/null +++ b/heat_integrationtests/requirements.stub @@ -0,0 +1,31 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +# These are stubs for requirements. This file must be processed with +# heat_integrationtests/install-requirements to generate and install the real +# requirements.txt where versions for packages present in +# main project requirements will be synced from main requirements files. + +pbr +kombu +os-collect-config +oslo.log +oslo.messaging +oslo.concurrency +oslo.config +oslo.utils +paramiko +python-ceilometerclient +python-cinderclient +python-keystoneclient +python-heatclient +python-neutronclient +python-novaclient +python-swiftclient +PyYAML +requests +six +testrepository +testscenarios +testtools diff --git a/heat_integrationtests/requirements.txt b/heat_integrationtests/requirements.txt deleted file mode 100644 index 3e24571411..0000000000 --- a/heat_integrationtests/requirements.txt +++ /dev/null @@ -1,25 +0,0 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. -pbr<2.0,>=0.11 -kombu>=3.0.7 -os-collect-config # Apache-2.0 -oslo.log>=1.2.0 # Apache-2.0 -oslo.messaging!=1.12.0,>=1.8.0 # Apache-2.0 -oslo.concurrency>=2.1.0 -oslo.config>=1.11.0 # Apache-2.0 -oslo.utils>=1.6.0 # Apache-2.0 -paramiko>=1.13.0 -python-ceilometerclient>=1.5.0 -python-cinderclient>=1.3.1 -python-keystoneclient>=1.6.0 -python-heatclient>=0.6.0 -python-neutronclient>=2.6.0 -python-novaclient>=2.29.0 -python-swiftclient>=2.2.0 -PyYAML>=3.1.0 -requests>=2.5.2 -six>=1.9.0 -testrepository>=0.0.18 -testscenarios>=0.4 -testtools>=1.4.0 diff --git a/test-requirements.txt b/test-requirements.txt index a20adba6f0..39c8f8d4d4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -12,7 +12,6 @@ mox3>=0.7.0 # Apache-2.0 PyMySQL>=0.6.2 # MIT License oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 -paramiko>=1.16.0 # LGPL qpid-python;python_version=='2.7' # Apache-2.0 psycopg2>=2.5 # LGPL/ZPL sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD @@ -21,3 +20,6 @@ testscenarios>=0.4 # Apache-2.0/BSD testtools>=1.4.0 # MIT testresources>=0.2.4 # Apache-2.0/BSD reno>=0.1.1 # Apache2 +# Next two are used in integration tests only +os-collect-config # Apache-2.0 +paramiko>=1.16.0 # LGPL diff --git a/tox.ini b/tox.ini index fce08e8935..5aa6adcf8f 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,8 @@ setenv = VIRTUAL_ENV={envdir} TESTR_START_DIR=heat_integrationtests passenv = OS_* usedevelop = False -deps = -r{toxinidir}/heat_integrationtests/requirements.txt +install_command = {toxinidir}/heat_integrationtests/install-requirements --stub {toxinidir}/heat_integrationtests/requirements.stub --source {toxinidir}/requirements.txt --source {toxinidir}/test-requirements.txt {packages} {opts} +deps = -t{toxinidir}/heat_integrationtests/requirements.txt commands = bash tools/pretty_tox.sh '{posargs}'