From 8e937d702398e9821a70843fa693131bee236339 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Fri, 14 Dec 2012 11:11:41 -0500 Subject: [PATCH] Fix use of venv in Tempest. Currently when running Tempest tests with run_tests.sh a venv isn't being used. This leaves tempest at the mercy of whatever required package versions are installed on the system. This patch takes install_venv.py and with_venv.sh from nova and integrates them into run_tests to ensure that the correct dependencies versions from pip-requires and test-requires are being used. Change-Id: I4bf4a02890a33c4034e4493d1763ed4019fdf46e --- .gitignore | 1 + run_tests.sh | 49 ++++++++- tools/install_venv.py | 248 ++++++++++++++++++++++++++++++++++++++++++ tools/pip-requires | 3 + tools/with_venv.sh | 4 +- 5 files changed, 301 insertions(+), 4 deletions(-) create mode 100644 tools/install_venv.py diff --git a/.gitignore b/.gitignore index 061c2ffe60..b4dca86d65 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,6 @@ include/swift_objects/swift_large *.swo *.egg-info .tox +.venv dist build diff --git a/run_tests.sh b/run_tests.sh index 7016a30b6f..af33f7cccf 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -4,6 +4,10 @@ function usage { echo "Usage: $0 [OPTION]..." echo "Run Tempest test suite" echo "" + echo " -V, --virtual-env Always use virtualenv. Install automatically if not present" + echo " -N, --no-virtual-env Don't use virtualenv. Run tests in local environment" + echo " -s, --no-site-packages Isolate the virtualenv from the global Python environment" + echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added." echo " -s, --smoke Only run smoke tests" echo " -w, --whitebox Only run whitebox tests" echo " -p, --pep8 Just run pep8" @@ -15,6 +19,10 @@ function usage { function process_option { case "$1" in -h|--help) usage;; + -V|--virtual-env) always_venv=1; never_venv=0;; + -N|--no-virtual-env) always_venv=0; never_venv=1;; + -s|--no-site-packages) no_site_packages=1;; + -f|--force) force=1;; -d|--debug) set -o xtrace;; -p|--pep8) let just_pep8=1;; -s|--smoke) noseargs="$noseargs --attr=type=smoke";; @@ -25,6 +33,14 @@ function process_option { noseargs="" just_pep8=0 +venv=.venv +with_venv=tools/with_venv.sh +always_venv=0 +never_venv=0 +no_site_packages=0 +force=0 +wrapper="" + export NOSE_WITH_OPENSTACK=1 export NOSE_OPENSTACK_COLOR=1 @@ -37,6 +53,9 @@ for arg in "$@"; do process_option $arg done +if [ $no_site_packages -eq 1 ]; then + installvenvopts="--no-site-packages" +fi # only add tempest default if we don't specify a test if [[ "x$noseargs" =~ "tempest" ]]; then @@ -47,7 +66,7 @@ fi function run_tests { - $NOSETESTS + ${wrapper} $NOSETESTS } function run_pep8 { @@ -58,7 +77,7 @@ function run_pep8 { ignore='--ignore=N4,E121,E122,E125,E126' - python tools/hacking.py ${ignore} ${srcfiles} + ${wrapper} python tools/hacking.py ${ignore} ${srcfiles} } NOSETESTS="nosetests $noseargs" @@ -68,6 +87,32 @@ if [ $just_pep8 -eq 1 ]; then exit fi +if [ $never_venv -eq 0 ] +then + # Remove the virtual environment if --force used + if [ $force -eq 1 ]; then + echo "Cleaning virtualenv..." + rm -rf ${venv} + fi + if [ -e ${venv} ]; then + wrapper="${with_venv}" + else + if [ $always_venv -eq 1 ]; then + # Automatically install the virtualenv + python tools/install_venv.py $installvenvopts + wrapper="${with_venv}" + else + echo -e "No virtual environment found...create one? (Y/n) \c" + read use_ve + if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then + # Install the virtualenv and run the test suite in it + python tools/install_venv.py $installvenvopts + wrapper=${with_venv} + fi + fi + fi +fi + run_tests || exit if [ -z "$noseargs" ]; then diff --git a/tools/install_venv.py b/tools/install_venv.py new file mode 100644 index 0000000000..42ed32c70d --- /dev/null +++ b/tools/install_venv.py @@ -0,0 +1,248 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2010 OpenStack, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Installation script for Tempest's development virtualenv.""" + +import optparse +import os +import subprocess +import sys + + +ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +VENV = os.path.join(ROOT, '.venv') +PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires') +TEST_REQUIRES = os.path.join(ROOT, 'tools', 'test-requires') +PY_VERSION = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) + + +def die(message, *args): + print >> sys.stderr, message % args + sys.exit(1) + + +def check_python_version(): + if sys.version_info < (2, 6): + die("Need Python Version >= 2.6") + + +def run_command_with_code(cmd, redirect_output=True, check_exit_code=True): + """Runs a command in an out-of-process shell. + + Returns the output of that command. Working directory is ROOT. + """ + if redirect_output: + stdout = subprocess.PIPE + else: + stdout = None + + proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout) + output = proc.communicate()[0] + if check_exit_code and proc.returncode != 0: + die('Command "%s" failed.\n%s', ' '.join(cmd), output) + return (output, proc.returncode) + + +def run_command(cmd, redirect_output=True, check_exit_code=True): + return run_command_with_code(cmd, redirect_output, check_exit_code)[0] + + +class Distro(object): + + def check_cmd(self, cmd): + return bool(run_command(['which', cmd], check_exit_code=False).strip()) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if self.check_cmd('easy_install'): + print 'Installing virtualenv via easy_install...', + if run_command(['easy_install', 'virtualenv']): + print 'Succeeded' + return + else: + print 'Failed' + + die('ERROR: virtualenv not found.\n\nTempest development' + ' requires virtualenv, please install it using your' + ' favorite package management tool') + + def post_process(self): + """Any distribution-specific post-processing gets done here. + + In particular, this is useful for applying patches to code inside + the venv. + """ + pass + + +class Fedora(Distro): + """This covers all Fedora-based distributions. + + Includes: Fedora, RHEL, CentOS, Scientific Linux""" + + def check_pkg(self, pkg): + return run_command_with_code(['rpm', '-q', pkg], + check_exit_code=False)[1] == 0 + + def yum_install(self, pkg, **kwargs): + print "Attempting to install '%s' via yum" % pkg + run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs) + + def apply_patch(self, originalfile, patchfile): + run_command(['patch', originalfile, patchfile]) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if not self.check_pkg('python-virtualenv'): + self.yum_install('python-virtualenv', check_exit_code=False) + + super(Fedora, self).install_virtualenv() + + def post_process(self): + """Workaround for a bug in eventlet. + + This currently affects RHEL6.1, but the fix can safely be + applied to all RHEL and Fedora distributions. + + This can be removed when the fix is applied upstream. + + Nova: https://bugs.launchpad.net/nova/+bug/884915 + Upstream: https://bitbucket.org/which_linden/eventlet/issue/89 + """ + + # Install "patch" program if it's not there + if not self.check_pkg('patch'): + self.yum_install('patch') + + # Apply the eventlet patch + self.apply_patch(os.path.join(VENV, 'lib', PY_VERSION, 'site-packages', + 'eventlet/green/subprocess.py'), + 'contrib/redhat-eventlet.patch') + + +def get_distro(): + if (os.path.exists('/etc/fedora-release') or + os.path.exists('/etc/redhat-release')): + return Fedora() + else: + return Distro() + + +def check_dependencies(): + get_distro().install_virtualenv() + + +def create_virtualenv(venv=VENV, no_site_packages=True): + """Creates the virtual environment and installs PIP. + + Creates the virtual environment and installs PIP only into the + virtual environment. + """ + print 'Creating venv...', + if no_site_packages: + run_command(['virtualenv', '-q', '--no-site-packages', VENV]) + else: + run_command(['virtualenv', '-q', VENV]) + print 'done.' + print 'Installing pip in virtualenv...', + if not run_command(['tools/with_venv.sh', 'easy_install', + 'pip>1.0']).strip(): + die("Failed to install pip.") + print 'done.' + + +def pip_install(*args): + run_command(['tools/with_venv.sh', + 'pip', 'install', '--upgrade'] + list(args), + redirect_output=False) + + +def install_dependencies(venv=VENV): + print 'Installing dependencies with pip (this can take a while)...' + + # First things first, make sure our venv has the latest pip and distribute. + # NOTE: we keep pip at version 1.1 since the most recent version causes + # the .venv creation to fail. See: + # https://bugs.launchpad.net/nova/+bug/1047120 + pip_install('pip==1.1') + pip_install('distribute') + + # Install greenlet by hand - just listing it in the requires file does not + # get it in stalled in the right order + pip_install('greenlet') + + pip_install('-r', PIP_REQUIRES) + pip_install('-r', TEST_REQUIRES) + + # Install tempest into the virtual_env. No more path munging! + run_command([os.path.join(venv, 'bin/python'), 'setup.py', 'develop']) + + +def post_process(): + get_distro().post_process() + + +def print_help(): + help = """ + Tempest development environment setup is complete. + + Tempest development uses virtualenv to track and manage Python dependencies + while in development and testing. + + To activate the Tempest virtualenv for the extent of your current shell + session you can run: + + $ source .venv/bin/activate + + Or, if you prefer, you can run commands in the virtualenv on a case by case + basis by running: + + $ tools/with_venv.sh + + Also, make test will automatically use the virtualenv. + """ + print help + + +def parse_args(): + """Parses command-line arguments.""" + parser = optparse.OptionParser() + parser.add_option("-n", "--no-site-packages", dest="no_site_packages", + default=False, action="store_true", + help="Do not inherit packages from global Python" + " install") + return parser.parse_args() + + +def main(argv): + (options, args) = parse_args() + check_python_version() + check_dependencies() + create_virtualenv(no_site_packages=options.no_site_packages) + install_dependencies() + post_process() + print_help() + +if __name__ == '__main__': + main(sys.argv) diff --git a/tools/pip-requires b/tools/pip-requires index 7877906430..6542a3ca41 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -6,3 +6,6 @@ lxml boto>=2.2.1 paramiko netaddr +python-glanceclient>=0.5.0,<2 +python-keystoneclient>=0.2.0 +python-novaclient>=2.10.0,<3 diff --git a/tools/with_venv.sh b/tools/with_venv.sh index 2e2b855c38..550c4774e5 100755 --- a/tools/with_venv.sh +++ b/tools/with_venv.sh @@ -1,4 +1,4 @@ #!/bin/bash TOOLS=`dirname $0` -VENV=$TOOLS/../.kong-venv -source $VENV/bin/activate && $@ +VENV=$TOOLS/../.venv +source $VENV/bin/activate && "$@"