Browse Source

Create integration tests requirements on the fly

The problem of integration tests was that they use their own
requirements.txt that is unable to auto-sync with global requirements.

This patch removes dedicated requirements.txt and replaces it with a
stub file listing names (and possibly versions) of the packages needed.
Then a special script parses this stub file, and for every package
that is present in main project requirements files it pulls the versions
from main project requirements, generating and installing requirements
for the integration tests on the fly.

This will help keeping requirements for the integration tests always in
sync with main project requirements.

Change-Id: Ie79338cc10cc101fbf15b51c7923e3a7b8e4fbb4
Closes-Bug: #1490866
changes/29/218929/11
Pavlo Shchelokovskyy 7 years ago committed by Sergey Kraynev
parent
commit
dd50ac6352
  1. 4
      .gitignore
  2. 66
      heat_integrationtests/install-requirements
  3. 31
      heat_integrationtests/requirements.stub
  4. 25
      heat_integrationtests/requirements.txt
  5. 4
      test-requirements.txt
  6. 3
      tox.ini

4
.gitignore vendored

@ -21,6 +21,8 @@ cover
.pydevproject
doc/source/api/
etc/heat/heat.conf.sample
# integration tests requirements are auto-generated from stub file
heat_integrationtests/requirements.txt
# Files created by releasenotes build
releasenotes/build
releasenotes/build

66
heat_integrationtests/install-requirements

@ -0,0 +1,66 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Generate and install requirements from stub file and source files."""
import argparse
import subprocess
import sys
import pip
parser = argparse.ArgumentParser(description='Sync requirements.')
parser.add_argument('--stub', metavar='STUBFILE',
required=True,
help="File with requirements stubs.")
parser.add_argument('--source', metavar='SOURCE',
required=True, action='append',
help="Source file to sync requirements from. "
"May be supplied several times.")
parser.add_argument('-t', '--target', metavar='TARGET',
required=True,
help="Target file to write synced requirements to.")
parser.add_argument('pipopts', metavar='PIP OPTIONS',
nargs=argparse.REMAINDER,
help='Options to pass to "pip install".')
args = parser.parse_args()
sources = {}
for requirements_file in args.source:
rqs = pip.req.req_file.parse_requirements(requirements_file,
session=False)
sources.update({s.name: s for s in rqs})
stubs = list(pip.req.req_file.parse_requirements(args.stub,
session=False))
reqs = []
for r in stubs:
if r.name in sources:
# safe-guard for future additions to stub file
if r.specifier:
sys.exit("ERROR: package '%(pkg)s' in stub file %(stub)s "
"has version specified but is also present "
"in provided sources requirements. "
"Please remove version from the stub file." % {
'pkg': r.name, 'stub': args.stub})
reqs.append(sources[r.name])
else:
reqs.append(r)
with open(args.target, 'w') as target:
target.write('\n'.join([str(r.req) for r in reqs]))
pip_install = ['pip', 'install', '-r', args.target]
pip_install.extend(args.pipopts)
sys.exit(subprocess.call(pip_install))

31
heat_integrationtests/requirements.stub

@ -0,0 +1,31 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
# These are stubs for requirements. This file must be processed with
# heat_integrationtests/install-requirements to generate and install the real
# requirements.txt where versions for packages present in
# main project requirements will be synced from main requirements files.
pbr
kombu
os-collect-config
oslo.log
oslo.messaging
oslo.concurrency
oslo.config
oslo.utils
paramiko
python-ceilometerclient
python-cinderclient
python-keystoneclient
python-heatclient
python-neutronclient
python-novaclient
python-swiftclient
PyYAML
requests
six
testrepository
testscenarios
testtools

25
heat_integrationtests/requirements.txt

@ -1,25 +0,0 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
pbr<2.0,>=0.11
kombu>=3.0.7
os-collect-config # Apache-2.0
oslo.log>=1.2.0 # Apache-2.0
oslo.messaging!=1.12.0,>=1.8.0 # Apache-2.0
oslo.concurrency>=2.1.0
oslo.config>=1.11.0 # Apache-2.0
oslo.utils>=1.6.0 # Apache-2.0
paramiko>=1.13.0
python-ceilometerclient>=1.5.0
python-cinderclient>=1.3.1
python-keystoneclient>=1.6.0
python-heatclient>=0.6.0
python-neutronclient>=2.6.0
python-novaclient>=2.29.0
python-swiftclient>=2.2.0
PyYAML>=3.1.0
requests>=2.5.2
six>=1.9.0
testrepository>=0.0.18
testscenarios>=0.4
testtools>=1.4.0

4
test-requirements.txt

@ -12,7 +12,6 @@ mox3>=0.7.0 # Apache-2.0
PyMySQL>=0.6.2 # MIT License
oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
oslotest>=1.10.0 # Apache-2.0
paramiko>=1.16.0 # LGPL
qpid-python;python_version=='2.7' # Apache-2.0
psycopg2>=2.5 # LGPL/ZPL
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD
@ -21,3 +20,6 @@ testscenarios>=0.4 # Apache-2.0/BSD
testtools>=1.4.0 # MIT
testresources>=0.2.4 # Apache-2.0/BSD
reno>=0.1.1 # Apache2
# Next two are used in integration tests only
os-collect-config # Apache-2.0
paramiko>=1.16.0 # LGPL

3
tox.ini

@ -28,7 +28,8 @@ setenv = VIRTUAL_ENV={envdir}
TESTR_START_DIR=heat_integrationtests
passenv = OS_*
usedevelop = False
deps = -r{toxinidir}/heat_integrationtests/requirements.txt
install_command = {toxinidir}/heat_integrationtests/install-requirements --stub {toxinidir}/heat_integrationtests/requirements.stub --source {toxinidir}/requirements.txt --source {toxinidir}/test-requirements.txt {packages} {opts}
deps = -t{toxinidir}/heat_integrationtests/requirements.txt
commands =
bash tools/pretty_tox.sh '{posargs}'

Loading…
Cancel
Save