From 0ac71edcf401bc917ed45d5e47a41e464da6780d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 13 Feb 2012 15:16:54 -0800 Subject: [PATCH] First attempt at adding devstack2 tests. --- run_tests.py | 92 ++++++++++++++++++++ run_tests.sh | 193 +++++++++++++++++++++++++++++++++++++++++ tools/install_venv.py | 142 ++++++++++++++++++++++++++++++ tools/pip-requires | 14 +++ tools/validate_json.py | 118 +++++++++++++++++++++++++ tools/with_venv.sh | 4 + 6 files changed, 563 insertions(+) create mode 100755 run_tests.py create mode 100755 run_tests.sh create mode 100644 tools/install_venv.py create mode 100644 tools/pip-requires create mode 100644 tools/validate_json.py create mode 100755 tools/with_venv.sh diff --git a/run_tests.py b/run_tests.py new file mode 100755 index 00000000..85f249fa --- /dev/null +++ b/run_tests.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +""" +To run all tests + python run_tests.py + +To run a single test: + python run_tests.py + functional.test_extensions:TestExtensions.test_extensions_json + +To run a single test module: + python run_tests.py functional.test_extensions + +""" +import logging +import os +import sys +import subprocess + +logger = logging.getLogger(__name__) + +TESTS = [ +] + + +def parse_suite_filter(): + """ Parses out -O or --only argument and returns the value after it as the + filter. Removes it from sys.argv in the process. """ + + filter = None + if '-O' in sys.argv or '--only' in sys.argv: + for i in range(len(sys.argv)): + if sys.argv[i] in ['-O', '--only']: + if len(sys.argv) > i + 1: + # Remove -O/--only settings from sys.argv + sys.argv.pop(i) + filter = sys.argv.pop(i) + break + return filter + + +if __name__ == '__main__': + filter = parse_suite_filter() + if filter: + TESTS = [t for t in TESTS if filter in str(t)] + if not TESTS: + print 'No test configuration by the name %s found' % filter + sys.exit(2) + #Run test suites + if len(TESTS) > 1: + directory = os.getcwd() + for test_num, test_cls in enumerate(TESTS): + try: + result = test_cls().run() + if result: + logger.error("Run returned %s for test %s. Exiting" % + (result, test_cls.__name__)) + sys.exit(result) + except Exception, e: + print "Error:", e + logger.exception(e) + sys.exit(1) + # Collect coverage from each run. They'll be combined later in .sh + if '--with-coverage' in sys.argv: + coverage_file = os.path.join(directory, ".coverage") + target_file = "%s.%s" % (coverage_file, test_cls.__name__) + try: + if os.path.exists(target_file): + logger.info("deleting %s" % target_file) + os.unlink(target_file) + if os.path.exists(coverage_file): + logger.info("Saving %s to %s" % (coverage_file, + target_file)) + os.rename(coverage_file, target_file) + except Exception, e: + logger.exception(e) + print ("Failed to move coverage file while running test" + ": %s. Error reported was: %s" % + (test_cls.__name__, e)) + sys.exit(1) + else: + for test_num, test_cls in enumerate(TESTS): + try: + result = test_cls().run() + if result: + logger.error("Run returned %s for test %s. Exiting" % + (result, test_cls.__name__)) + sys.exit(result) + except Exception, e: + print "Error:", e + logger.exception(e) + sys.exit(1) diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 00000000..6baeb5c6 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,193 @@ +#!/bin/bash + +set -eu + +function usage { + echo "Usage: $0 [OPTION]..." + echo "Run Devstacks's test suite(s)" + echo "" + echo " -O, --only test_suite Only run the specified test suite. Valid values are:" + echo " Note: by default, run_tests will run all suites" + echo " -V, --virtual-env Always use virtualenv. Install automatically if not present" + echo " -N, --no-virtual-env Don't use virtualenv. Run tests in local environment" + echo " -x, --stop Stop running tests after the first error or failure." + echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added." + echo " Note: you might need to 'sudo' this since it pip installs into the vitual environment" + echo " -P, --skip-pep8 Just run tests; skip pep8 check" + echo " -p, --pep8 Just run pep8" + echo " -l, --pylint Just run pylint" + echo " -j, --json Just validate JSON" + echo " -c, --with-coverage Generate coverage report" + echo " -h, --help Print this usage message" + echo " --hide-elapsed Don't print the elapsed time for each test along with slow test list" + echo " --verbose Print additional logging" + echo "" + echo "Note: with no options specified, the script will try to run the tests in a virtual environment," + echo " If no virtualenv is found, the script will ask if you would like to create one. If you " + echo " prefer to run tests NOT in a virtual environment, simply pass the -N option." + echo "" + echo "Note: with no options specified, the script will run the pep8 check after completing the tests." + echo " If you prefer not to run pep8, simply pass the -P option." + exit +} + +only_run_flag=0 +only_run="" +function process_option { + if [ $only_run_flag -eq 1 ]; then + only_run_flag=0 + only_run=$1 + return + else + case "$1" in + -h|--help) usage;; + -V|--virtual-env) always_venv=1; never_venv=0;; + -N|--no-virtual-env) always_venv=0; never_venv=1;; + -O|--only) only_run_flag=1;; + -f|--force) force=1;; + -P|--skip-pep8) skip_pep8=1;; + -p|--pep8) just_pep8=1;; + -l|--pylint) just_pylint=1;; + -j|--json) just_json=1;; + -c|--with-coverage) coverage=1;; + -*) addlopts="$addlopts $1";; + *) addlargs="$addlargs $1" + esac + fi +} + +venv=.venv +with_venv=tools/with_venv.sh +always_venv=0 +never_venv=0 +force=0 +addlargs= +addlopts= +wrapper="" +just_pep8=0 +skip_pep8=0 +just_pylint=0 +just_json=0 +coverage=0 + +for arg in "$@"; do + process_option $arg +done + +# If enabled, tell nose/unittest to collect coverage data +if [ $coverage -eq 1 ]; then + addlopts="$addlopts --with-coverage --cover-package=devstack" +fi + +if [ "x$only_run" = "x" ]; then + RUNTESTS="python run_tests.py$addlopts$addlargs" +else + RUNTESTS="python run_tests.py$addlopts$addlargs -O $only_run" +fi + +if [ $never_venv -eq 0 ] +then + # Remove the virtual environment if --force used + if [ $force -eq 1 ]; then + echo "Cleaning virtualenv..." + rm -rf ${venv} + fi + if [ -e ${venv} ]; then + wrapper="${with_venv}" + else + if [ $always_venv -eq 1 ]; then + # Automatically install the virtualenv + python tools/install_venv.py + wrapper="${with_venv}" + else + echo -e "No virtual environment found...create one? (Y/n) \c" + read use_ve + if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then + # Install the virtualenv and run the test suite in it + python tools/install_venv.py + wrapper=${with_venv} + fi + fi + fi +fi + +function run_tests { + # Just run the test suites in current environment + ${wrapper} $RUNTESTS 2> run_tests.log + # If we get some short import error right away, print the error log directly + RESULT=$? + if [ "$RESULT" -ne "0" ]; + then + ERRSIZE=`wc -l run_tests.log | awk '{print \$1}'` + if [ "$ERRSIZE" -lt "40" ]; + then + cat run_tests.log + fi + fi + return $RESULT +} + +function run_pep8 { + echo "Running pep8 ..." + # Opt-out files from pep8 + ignore_scripts="*.sh" + ignore_files="*pip-requires,*.log" + ignore_dirs="*tools*" + GLOBIGNORE="$ignore_scripts,$ignore_files,$ignore_dirs" + srcfiles=`find bin -type f -not -name "*.log" -not -name "*.db"` + srcfiles+=" devstack run_tests.py" + # Just run PEP8 in current environment + ${wrapper} pep8 --repeat --show-pep8 --show-source \ + --exclude=$GLOBIGNORE ${srcfiles} +} + +function run_pylint { + echo "Running pylint ..." + PYLINT_OPTIONS="--rcfile=pylintrc --output-format=parseable" + PYLINT_INCLUDE="devstack" + echo "Pylint messages count: " + pylint $PYLINT_OPTIONS $PYLINT_INCLUDE | grep 'keystone/' | wc -l + echo "Run 'pylint $PYLINT_OPTIONS $PYLINT_INCLUDE' for a full report." +} + +function validate_json { + echo "Validating JSON..." + python tools/validate_json.py +} + + +# Delete old coverage data from previous runs +if [ $coverage -eq 1 ]; then + ${wrapper} coverage erase +fi + +if [ $just_pep8 -eq 1 ]; then + run_pep8 + exit +fi + +if [ $just_pylint -eq 1 ]; then + run_pylint + exit +fi + +if [ $just_json -eq 1 ]; then + validate_json + exit +fi + + +run_tests +if [ $skip_pep8 -eq 0 ]; then + # Run the pep8 check + run_pep8 +fi + +# Since we run multiple test suites, we need to execute 'coverage combine' +if [ $coverage -eq 1 ]; then + echo "Generating coverage report in covhtml/" + ${wrapper} coverage combine + ${wrapper} coverage html -d covhtml -i + ${wrapper} coverage report --omit='/usr*,keystone/test*,.,setup.py,*egg*,/Library*,*.xml,*.tpl' +fi + diff --git a/tools/install_venv.py b/tools/install_venv.py new file mode 100644 index 00000000..9f1a900a --- /dev/null +++ b/tools/install_venv.py @@ -0,0 +1,142 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2010 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Installation script for Keystone's development virtualenv +""" + +import os +import subprocess +import sys + + +ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +VENV = os.path.join(ROOT, '.venv') +PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires') + + +def die(message, *args): + print >> sys.stderr, message % args + sys.exit(1) + + +def run_command(cmd, redirect_output=True, check_exit_code=True): + """ + Runs a command in an out-of-process shell, returning the + output of that command. Working directory is ROOT. + """ + if redirect_output: + stdout = subprocess.PIPE + else: + stdout = None + + proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout) + output = proc.communicate()[0] + if check_exit_code and proc.returncode != 0: + die('Command "%s" failed.\n%s', ' '.join(cmd), output) + return output + + +HAS_EASY_INSTALL = bool(run_command(['which', 'easy_install'], + check_exit_code=False).strip()) +HAS_VIRTUALENV = bool(run_command(['which', 'virtualenv'], + check_exit_code=False).strip()) + + +def check_dependencies(): + """Make sure virtualenv is in the path.""" + + if not HAS_VIRTUALENV: + print 'not found.' + # Try installing it via easy_install... + if HAS_EASY_INSTALL: + print 'Installing virtualenv via easy_install...', + if not run_command(['which', 'easy_install']): + die('ERROR: virtualenv not found.\n\n' + 'Keystone development requires virtualenv, please install' + ' it using your favorite package management tool') + print 'done.' + print 'done.' + + +def create_virtualenv(venv=VENV): + """ + Creates the virtual environment and installs PIP only into the + virtual environment + """ + print 'Creating venv...', + run_command(['virtualenv', '-q', '--no-site-packages', VENV]) + print 'done.' + print 'Installing pip in virtualenv...', + if not run_command(['tools/with_venv.sh', 'easy_install', + 'pip>1.0']).strip(): + die("Failed to install pip.") + print 'done.' + + +def install_dependencies(venv=VENV): + print 'Installing dependencies with pip (this can take a while)...' + + # Install greenlet by hand - just listing it in the requires file does not + # get it in stalled in the right order + venv_tool = 'tools/with_venv.sh' + run_command([venv_tool, 'pip', 'install', '-E', venv, '-r', PIP_REQUIRES], + redirect_output=False) + + # Tell the virtual env how to "import keystone" + + for version in ['python2.7', 'python2.6']: + pth = os.path.join(venv, "lib", version, "site-packages") + if os.path.exists(pth): + pthfile = os.path.join(pth, "keystone.pth") + f = open(pthfile, 'w') + f.write("%s\n" % ROOT) + + +def print_help(): + help = """ + Keystone development environment setup is complete. + + Keystone development uses virtualenv to track and manage Python dependencies + while in development and testing. + + To activate the Keystone virtualenv for the extent of your current shell + session you can run: + + $ source .venv/bin/activate + + Or, if you prefer, you can run commands in the virtualenv on a case by case + basis by running: + + $ tools/with_venv.sh + + Also, make test will automatically use the virtualenv. + """ + print help + + +def main(argv): + check_dependencies() + create_virtualenv() + install_dependencies() + print_help() + +if __name__ == '__main__': + main(sys.argv) diff --git a/tools/pip-requires b/tools/pip-requires new file mode 100644 index 00000000..631245f0 --- /dev/null +++ b/tools/pip-requires @@ -0,0 +1,14 @@ +# NOTE: You may need to install additional binary packages prior to using 'pip install' +# See the Contributor Documentation for more information + +# Development +netifaces +termcolor + +# Testing +nose # for test discovery and console feedback +unittest2 # backport of unittest lib in python 2.7 +pylint # static code analysis +pep8 # checks for PEP8 code style compliance +mox # mock object framework +coverage # computes code coverage percentages diff --git a/tools/validate_json.py b/tools/validate_json.py new file mode 100644 index 00000000..24031992 --- /dev/null +++ b/tools/validate_json.py @@ -0,0 +1,118 @@ +""" +Searches the given path for JSON files, and validates their contents. + +Optionally, replaces valid JSON files with their pretty-printed +counterparts. +""" + +import argparse +import collections +import errno +import json +import logging +import os +import re + + +# Configure logging +logging.basicConfig(format='%(levelname)s: %(message)s') + +# Configure commandlineability +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('-p', type=str, required=False, default='.', + help='the path to search for JSON files', dest='path') +parser.add_argument('-r', type=str, required=False, default='.json$', + help='the regular expression to match filenames against ' \ + '(not absolute paths)', dest='regexp') +args = parser.parse_args() + + +def main(): + files = find_matching_files(args.path, args.regexp) + + results = True + for path in files: + results &= validate_json(path) + + # Invert our test results to produce a status code + exit(not results) + + +def validate_json(path): + """Open a file and validate it's contents as JSON""" + try: + contents = read_file(path) + + if contents is False: + logging.warning('Insufficient permissions to open: %s' % path) + return False + except: + logging.warning('Unable to open: %s' % path) + return False + + #knock off comments + ncontents = list() + for line in contents.splitlines(): + tmp_line = line.strip() + if tmp_line.startswith("#"): + continue + else: + ncontents.append(line) + + contents = os.linesep.join(ncontents) + try: + ordered_dict = json.loads(contents, + object_pairs_hook=collections.OrderedDict) + except: + logging.error('Unable to parse: %s' % path) + return False + + return True + + +def find_matching_files(path, pattern): + """Search the given path for files matching the given pattern""" + + regex = re.compile(pattern) + + json_files = [] + for root, dirs, files in os.walk(path): + for name in files: + if regex.search(name): + full_name = os.path.join(root, name) + json_files.append(full_name) + return json_files + + +def read_file(path): + """Attempt to read a file safely + + Returns the contents of the file as a string on success, False otherwise""" + try: + fp = open(path) + except IOError as e: + if e.errno == errno.EACCES: + # permission error + return False + raise + else: + with fp: + return fp.read() + + +def replace_file(path, new_contents): + """Overwrite the file at the given path with the new contents + + Returns True on success, False otherwise.""" + try: + f = open(path, 'w') + f.write(new_contents) + f.close() + except: + logging.error('Unable to write: %s' % f) + return False + return True + + +if __name__ == "__main__": + main() diff --git a/tools/with_venv.sh b/tools/with_venv.sh new file mode 100755 index 00000000..c8d2940f --- /dev/null +++ b/tools/with_venv.sh @@ -0,0 +1,4 @@ +#!/bin/bash +TOOLS=`dirname $0` +VENV=$TOOLS/../.venv +source $VENV/bin/activate && $@