Fix use of venv in Tempest.

Currently when running Tempest tests with run_tests.sh a venv
isn't being used. This leaves tempest at the mercy of whatever
required package versions are installed on the system.

This patch takes install_venv.py and with_venv.sh from nova and
integrates them into run_tests to ensure that the correct
dependencies versions from pip-requires and test-requires are
being used.

Change-Id: I4bf4a02890a33c4034e4493d1763ed4019fdf46e
This commit is contained in:
Matthew Treinish 2012-12-14 11:11:41 -05:00
parent e7b9be66bc
commit 8e937d7023
5 changed files with 301 additions and 4 deletions

1
.gitignore vendored
View File

@ -8,5 +8,6 @@ include/swift_objects/swift_large
*.swo
*.egg-info
.tox
.venv
dist
build

View File

@ -4,6 +4,10 @@ function usage {
echo "Usage: $0 [OPTION]..."
echo "Run Tempest test suite"
echo ""
echo " -V, --virtual-env Always use virtualenv. Install automatically if not present"
echo " -N, --no-virtual-env Don't use virtualenv. Run tests in local environment"
echo " -s, --no-site-packages Isolate the virtualenv from the global Python environment"
echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added."
echo " -s, --smoke Only run smoke tests"
echo " -w, --whitebox Only run whitebox tests"
echo " -p, --pep8 Just run pep8"
@ -15,6 +19,10 @@ function usage {
function process_option {
case "$1" in
-h|--help) usage;;
-V|--virtual-env) always_venv=1; never_venv=0;;
-N|--no-virtual-env) always_venv=0; never_venv=1;;
-s|--no-site-packages) no_site_packages=1;;
-f|--force) force=1;;
-d|--debug) set -o xtrace;;
-p|--pep8) let just_pep8=1;;
-s|--smoke) noseargs="$noseargs --attr=type=smoke";;
@ -25,6 +33,14 @@ function process_option {
noseargs=""
just_pep8=0
venv=.venv
with_venv=tools/with_venv.sh
always_venv=0
never_venv=0
no_site_packages=0
force=0
wrapper=""
export NOSE_WITH_OPENSTACK=1
export NOSE_OPENSTACK_COLOR=1
@ -37,6 +53,9 @@ for arg in "$@"; do
process_option $arg
done
if [ $no_site_packages -eq 1 ]; then
installvenvopts="--no-site-packages"
fi
# only add tempest default if we don't specify a test
if [[ "x$noseargs" =~ "tempest" ]]; then
@ -47,7 +66,7 @@ fi
function run_tests {
$NOSETESTS
${wrapper} $NOSETESTS
}
function run_pep8 {
@ -58,7 +77,7 @@ function run_pep8 {
ignore='--ignore=N4,E121,E122,E125,E126'
python tools/hacking.py ${ignore} ${srcfiles}
${wrapper} python tools/hacking.py ${ignore} ${srcfiles}
}
NOSETESTS="nosetests $noseargs"
@ -68,6 +87,32 @@ if [ $just_pep8 -eq 1 ]; then
exit
fi
if [ $never_venv -eq 0 ]
then
# Remove the virtual environment if --force used
if [ $force -eq 1 ]; then
echo "Cleaning virtualenv..."
rm -rf ${venv}
fi
if [ -e ${venv} ]; then
wrapper="${with_venv}"
else
if [ $always_venv -eq 1 ]; then
# Automatically install the virtualenv
python tools/install_venv.py $installvenvopts
wrapper="${with_venv}"
else
echo -e "No virtual environment found...create one? (Y/n) \c"
read use_ve
if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then
# Install the virtualenv and run the test suite in it
python tools/install_venv.py $installvenvopts
wrapper=${with_venv}
fi
fi
fi
fi
run_tests || exit
if [ -z "$noseargs" ]; then

248
tools/install_venv.py Normal file
View File

@ -0,0 +1,248 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Copyright 2010 OpenStack, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Installation script for Tempest's development virtualenv."""
import optparse
import os
import subprocess
import sys
ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
VENV = os.path.join(ROOT, '.venv')
PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires')
TEST_REQUIRES = os.path.join(ROOT, 'tools', 'test-requires')
PY_VERSION = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
def die(message, *args):
print >> sys.stderr, message % args
sys.exit(1)
def check_python_version():
if sys.version_info < (2, 6):
die("Need Python Version >= 2.6")
def run_command_with_code(cmd, redirect_output=True, check_exit_code=True):
"""Runs a command in an out-of-process shell.
Returns the output of that command. Working directory is ROOT.
"""
if redirect_output:
stdout = subprocess.PIPE
else:
stdout = None
proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout)
output = proc.communicate()[0]
if check_exit_code and proc.returncode != 0:
die('Command "%s" failed.\n%s', ' '.join(cmd), output)
return (output, proc.returncode)
def run_command(cmd, redirect_output=True, check_exit_code=True):
return run_command_with_code(cmd, redirect_output, check_exit_code)[0]
class Distro(object):
def check_cmd(self, cmd):
return bool(run_command(['which', cmd], check_exit_code=False).strip())
def install_virtualenv(self):
if self.check_cmd('virtualenv'):
return
if self.check_cmd('easy_install'):
print 'Installing virtualenv via easy_install...',
if run_command(['easy_install', 'virtualenv']):
print 'Succeeded'
return
else:
print 'Failed'
die('ERROR: virtualenv not found.\n\nTempest development'
' requires virtualenv, please install it using your'
' favorite package management tool')
def post_process(self):
"""Any distribution-specific post-processing gets done here.
In particular, this is useful for applying patches to code inside
the venv.
"""
pass
class Fedora(Distro):
"""This covers all Fedora-based distributions.
Includes: Fedora, RHEL, CentOS, Scientific Linux"""
def check_pkg(self, pkg):
return run_command_with_code(['rpm', '-q', pkg],
check_exit_code=False)[1] == 0
def yum_install(self, pkg, **kwargs):
print "Attempting to install '%s' via yum" % pkg
run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs)
def apply_patch(self, originalfile, patchfile):
run_command(['patch', originalfile, patchfile])
def install_virtualenv(self):
if self.check_cmd('virtualenv'):
return
if not self.check_pkg('python-virtualenv'):
self.yum_install('python-virtualenv', check_exit_code=False)
super(Fedora, self).install_virtualenv()
def post_process(self):
"""Workaround for a bug in eventlet.
This currently affects RHEL6.1, but the fix can safely be
applied to all RHEL and Fedora distributions.
This can be removed when the fix is applied upstream.
Nova: https://bugs.launchpad.net/nova/+bug/884915
Upstream: https://bitbucket.org/which_linden/eventlet/issue/89
"""
# Install "patch" program if it's not there
if not self.check_pkg('patch'):
self.yum_install('patch')
# Apply the eventlet patch
self.apply_patch(os.path.join(VENV, 'lib', PY_VERSION, 'site-packages',
'eventlet/green/subprocess.py'),
'contrib/redhat-eventlet.patch')
def get_distro():
if (os.path.exists('/etc/fedora-release') or
os.path.exists('/etc/redhat-release')):
return Fedora()
else:
return Distro()
def check_dependencies():
get_distro().install_virtualenv()
def create_virtualenv(venv=VENV, no_site_packages=True):
"""Creates the virtual environment and installs PIP.
Creates the virtual environment and installs PIP only into the
virtual environment.
"""
print 'Creating venv...',
if no_site_packages:
run_command(['virtualenv', '-q', '--no-site-packages', VENV])
else:
run_command(['virtualenv', '-q', VENV])
print 'done.'
print 'Installing pip in virtualenv...',
if not run_command(['tools/with_venv.sh', 'easy_install',
'pip>1.0']).strip():
die("Failed to install pip.")
print 'done.'
def pip_install(*args):
run_command(['tools/with_venv.sh',
'pip', 'install', '--upgrade'] + list(args),
redirect_output=False)
def install_dependencies(venv=VENV):
print 'Installing dependencies with pip (this can take a while)...'
# First things first, make sure our venv has the latest pip and distribute.
# NOTE: we keep pip at version 1.1 since the most recent version causes
# the .venv creation to fail. See:
# https://bugs.launchpad.net/nova/+bug/1047120
pip_install('pip==1.1')
pip_install('distribute')
# Install greenlet by hand - just listing it in the requires file does not
# get it in stalled in the right order
pip_install('greenlet')
pip_install('-r', PIP_REQUIRES)
pip_install('-r', TEST_REQUIRES)
# Install tempest into the virtual_env. No more path munging!
run_command([os.path.join(venv, 'bin/python'), 'setup.py', 'develop'])
def post_process():
get_distro().post_process()
def print_help():
help = """
Tempest development environment setup is complete.
Tempest development uses virtualenv to track and manage Python dependencies
while in development and testing.
To activate the Tempest virtualenv for the extent of your current shell
session you can run:
$ source .venv/bin/activate
Or, if you prefer, you can run commands in the virtualenv on a case by case
basis by running:
$ tools/with_venv.sh <your command>
Also, make test will automatically use the virtualenv.
"""
print help
def parse_args():
"""Parses command-line arguments."""
parser = optparse.OptionParser()
parser.add_option("-n", "--no-site-packages", dest="no_site_packages",
default=False, action="store_true",
help="Do not inherit packages from global Python"
" install")
return parser.parse_args()
def main(argv):
(options, args) = parse_args()
check_python_version()
check_dependencies()
create_virtualenv(no_site_packages=options.no_site_packages)
install_dependencies()
post_process()
print_help()
if __name__ == '__main__':
main(sys.argv)

View File

@ -6,3 +6,6 @@ lxml
boto>=2.2.1
paramiko
netaddr
python-glanceclient>=0.5.0,<2
python-keystoneclient>=0.2.0
python-novaclient>=2.10.0,<3

View File

@ -1,4 +1,4 @@
#!/bin/bash
TOOLS=`dirname $0`
VENV=$TOOLS/../.kong-venv
source $VENV/bin/activate && $@
VENV=$TOOLS/../.venv
source $VENV/bin/activate && "$@"