diff --git a/.mailmap b/.mailmap new file mode 100644 index 000000000..c3f1f5615 --- /dev/null +++ b/.mailmap @@ -0,0 +1,2 @@ + + diff --git a/README.rst b/README.rst index b8e26c9fa..1b2bdec1b 100644 --- a/README.rst +++ b/README.rst @@ -98,7 +98,6 @@ Additional command-line options and their associated environment variables are listed here:: --debug # turns on some debugging of the API conversation - (via httplib2) --verbose | -v # Increase verbosity of output. Can be repeated. --quiet | -q # suppress output except warnings and errors --help | -h # show a help message and exit diff --git a/openstackclient/common/clientmanager.py b/openstackclient/common/clientmanager.py index 73c2e5708..4d0492130 100644 --- a/openstackclient/common/clientmanager.py +++ b/openstackclient/common/clientmanager.py @@ -20,7 +20,6 @@ import logging -from openstackclient.common import exceptions as exc from openstackclient.compute import client as compute_client from openstackclient.identity import client as identity_client from openstackclient.image import client as image_client diff --git a/openstackclient/common/openstackkeyring.py b/openstackclient/common/openstackkeyring.py index 3a5ce27f8..2b03e7534 100644 --- a/openstackclient/common/openstackkeyring.py +++ b/openstackclient/common/openstackkeyring.py @@ -21,7 +21,6 @@ Keyring backend for Openstack, to store encrypted password in a file. from Crypto.Cipher import AES -import crypt import keyring import os diff --git a/openstackclient/common/utils.py b/openstackclient/common/utils.py index 70555be5f..c099889d2 100644 --- a/openstackclient/common/utils.py +++ b/openstackclient/common/utils.py @@ -23,8 +23,6 @@ import os import sys import uuid -import prettytable - from openstackclient.common import exceptions diff --git a/openstackclient/compute/client.py b/openstackclient/compute/client.py index 3c17b17ad..bef9f949d 100644 --- a/openstackclient/compute/client.py +++ b/openstackclient/compute/client.py @@ -17,7 +17,6 @@ import logging -from openstackclient.common import exceptions as exc from openstackclient.common import utils LOG = logging.getLogger(__name__) diff --git a/openstackclient/compute/v2/server.py b/openstackclient/compute/v2/server.py index e496bd459..366d7640e 100644 --- a/openstackclient/compute/v2/server.py +++ b/openstackclient/compute/v2/server.py @@ -21,7 +21,6 @@ Server action implementations import logging import os -import sys import time from cliff import command diff --git a/openstackclient/identity/client.py b/openstackclient/identity/client.py index b7066e5e6..3d58ad7ba 100644 --- a/openstackclient/identity/client.py +++ b/openstackclient/identity/client.py @@ -17,7 +17,6 @@ import logging -from openstackclient.common import exceptions as exc from openstackclient.common import utils LOG = logging.getLogger(__name__) diff --git a/openstackclient/openstack/common/setup.py b/openstackclient/openstack/common/setup.py index e6f72f034..e960002d2 100644 --- a/openstackclient/openstack/common/setup.py +++ b/openstackclient/openstack/common/setup.py @@ -78,10 +78,6 @@ def parse_requirements(requirements_files=['requirements.txt', # -f lines are for index locations, and don't get used here elif re.match(r'\s*-f\s+', line): pass - # argparse is part of the standard library starting with 2.7 - # adding it to the requirements list screws distro installs - elif line == 'argparse' and sys.version_info >= (2, 7): - pass else: requirements.append(line) diff --git a/run_tests.sh b/run_tests.sh index b9252cac7..4700c1164 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,49 +1,152 @@ #!/bin/bash +set -eu + function usage { echo "Usage: $0 [OPTION]..." - echo "Run python-openstackclient's test suite(s)" + echo "Run python-openstackclient test suite" echo "" + echo " -V, --virtual-env Always use virtualenv. Install automatically if not present" + echo " -N, --no-virtual-env Don't use virtualenv. Run tests in local environment" + echo " -s, --no-site-packages Isolate the virtualenv from the global Python environment" + echo " -x, --stop Stop running tests after the first error or failure." + echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added." echo " -p, --pep8 Just run pep8" + echo " -P, --no-pep8 Don't run pep8" + echo " -c, --coverage Generate coverage report" echo " -h, --help Print this usage message" + echo " --hide-elapsed Don't print the elapsed time for each test along with slow test list" echo "" - echo "This script is deprecated and currently retained for compatibility." - echo 'You can run the full test suite for multiple environments by running "tox".' - echo 'You can run tests for only python 2.7 by running "tox -e py27", or run only' - echo 'the pep8 tests with "tox -e pep8".' + echo "Note: with no options specified, the script will try to run the tests in a virtual environment," + echo " If no virtualenv is found, the script will ask if you would like to create one. If you " + echo " prefer to run tests NOT in a virtual environment, simply pass the -N option." exit } -command -v tox > /dev/null 2>&1 -if [ $? -ne 0 ]; then - echo 'This script requires "tox" to run.' - echo 'You can install it with "pip install tox".' - exit 1; -fi - -just_pep8=0 - function process_option { case "$1" in -h|--help) usage;; - -p|--pep8) let just_pep8=1;; + -V|--virtual-env) always_venv=1; never_venv=0;; + -N|--no-virtual-env) always_venv=0; never_venv=1;; + -s|--no-site-packages) no_site_packages=1;; + -f|--force) force=1;; + -p|--pep8) just_pep8=1;; + -P|--no-pep8) no_pep8=1;; + -c|--coverage) coverage=1;; + -*) noseopts="$noseopts $1";; + *) noseargs="$noseargs $1" esac } +venv=.venv +with_venv=tools/with_venv.sh +always_venv=0 +never_venv=0 +force=0 +no_site_packages=0 +installvenvopts= +noseargs= +noseopts= +wrapper="" +just_pep8=0 +no_pep8=0 +coverage=0 + for arg in "$@"; do process_option $arg done +# If enabled, tell nose to collect coverage data +if [ $coverage -eq 1 ]; then + noseopts="$noseopts --with-coverage --cover-package=openstackclient" +fi + +if [ $no_site_packages -eq 1 ]; then + installvenvopts="--no-site-packages" +fi + +function run_tests { + # Just run the test suites in current environment + ${wrapper} $NOSETESTS + # If we get some short import error right away, print the error log directly + RESULT=$? + return $RESULT +} + +function run_pep8 { + echo "Running pep8 ..." + srcfiles="openstackclient tests" + # Just run PEP8 in current environment + # + # NOTE(sirp): W602 (deprecated 3-arg raise) is being ignored for the + # following reasons: + # + # 1. It's needed to preserve traceback information when re-raising + # exceptions; this is needed b/c Eventlet will clear exceptions when + # switching contexts. + # + # 2. There doesn't appear to be an alternative, "pep8-tool" compatible way of doing this + # in Python 2 (in Python 3 `with_traceback` could be used). + # + # 3. Can find no corroborating evidence that this is deprecated in Python 2 + # other than what the PEP8 tool claims. It is deprecated in Python 3, so, + # perhaps the mistake was thinking that the deprecation applied to Python 2 + # as well. + pep8_opts="--ignore=E202,W602 --repeat" + ${wrapper} pep8 ${pep8_opts} ${srcfiles} +} + +NOSETESTS="nosetests $noseopts $noseargs" + +if [ $never_venv -eq 0 ] +then + # Remove the virtual environment if --force used + if [ $force -eq 1 ]; then + echo "Cleaning virtualenv..." + rm -rf ${venv} + fi + if [ -e ${venv} ]; then + wrapper="${with_venv}" + else + if [ $always_venv -eq 1 ]; then + # Automatically install the virtualenv + python tools/install_venv.py $installvenvopts + wrapper="${with_venv}" + else + echo -e "No virtual environment found...create one? (Y/n) \c" + read use_ve + if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then + # Install the virtualenv and run the test suite in it + python tools/install_venv.py $installvenvopts + wrapper=${with_venv} + fi + fi + fi +fi + +# Delete old coverage data from previous runs +if [ $coverage -eq 1 ]; then + ${wrapper} coverage erase +fi + if [ $just_pep8 -eq 1 ]; then - tox -e pep8 - exit + run_pep8 + exit fi -tox -e py27 $toxargs 2>&1 | tee run_tests.err.log || exit -if [ ${PIPESTATUS[0]} -ne 0 ]; then - exit ${PIPESTATUS[0]} +run_tests + +# NOTE(sirp): we only want to run pep8 when we're running the full-test suite, +# not when we're running tests individually. To handle this, we need to +# distinguish between options (noseopts), which begin with a '-', and +# arguments (noseargs). +if [ -z "$noseargs" ]; then + if [ $no_pep8 -eq 0 ]; then + run_pep8 + fi fi -if [ -z "$toxargs" ]; then - tox -e pep8 +if [ $coverage -eq 1 ]; then + echo "Generating coverage report in covhtml/" + ${wrapper} coverage html -d covhtml -i fi diff --git a/tests/utils.py b/tests/utils.py index 792fe88ea..633442c92 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -2,7 +2,6 @@ import time -import mox import testtools @@ -10,12 +9,9 @@ class TestCase(testtools.TestCase): def setUp(self): super(TestCase, self).setUp() - self.mox = mox.Mox() self._original_time = time.time time.time = lambda: 1234 def tearDown(self): time.time = self._original_time - self.mox.UnsetStubs() - self.mox.VerifyAll() super(TestCase, self).tearDown() diff --git a/tools/install_venv.py b/tools/install_venv.py new file mode 100644 index 000000000..0957d26b1 --- /dev/null +++ b/tools/install_venv.py @@ -0,0 +1,244 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2010 OpenStack, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Installation script for python-openstackclient's development virtualenv +""" + +import optparse +import os +import subprocess +import sys +import platform + + +ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +VENV = os.path.join(ROOT, '.venv') +PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires') +TEST_REQUIRES = os.path.join(ROOT, 'tools', 'test-requires') +PY_VERSION = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) + + +def die(message, *args): + print >> sys.stderr, message % args + sys.exit(1) + + +def check_python_version(): + if sys.version_info < (2, 6): + die("Need Python Version >= 2.6") + + +def run_command_with_code(cmd, redirect_output=True, check_exit_code=True): + """ + Runs a command in an out-of-process shell, returning the + output of that command. Working directory is ROOT. + """ + if redirect_output: + stdout = subprocess.PIPE + else: + stdout = None + + proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout) + output = proc.communicate()[0] + if check_exit_code and proc.returncode != 0: + die('Command "%s" failed.\n%s', ' '.join(cmd), output) + return (output, proc.returncode) + + +def run_command(cmd, redirect_output=True, check_exit_code=True): + return run_command_with_code(cmd, redirect_output, check_exit_code)[0] + + +class Distro(object): + + def check_cmd(self, cmd): + return bool(run_command(['which', cmd], check_exit_code=False).strip()) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if self.check_cmd('easy_install'): + print 'Installing virtualenv via easy_install...', + if run_command(['easy_install', 'virtualenv']): + print 'Succeeded' + return + else: + print 'Failed' + + die('ERROR: virtualenv not found.\n\nDevelopment' + ' requires virtualenv, please install it using your' + ' favorite package management tool') + + def post_process(self): + """Any distribution-specific post-processing gets done here. + + In particular, this is useful for applying patches to code inside + the venv.""" + pass + + +class Debian(Distro): + """This covers all Debian-based distributions.""" + + def check_pkg(self, pkg): + return run_command_with_code(['dpkg', '-l', pkg], + check_exit_code=False)[1] == 0 + + def apt_install(self, pkg, **kwargs): + run_command(['sudo', 'apt-get', 'install', '-y', pkg], **kwargs) + + def apply_patch(self, originalfile, patchfile): + run_command(['patch', originalfile, patchfile]) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if not self.check_pkg('python-virtualenv'): + self.apt_install('python-virtualenv', check_exit_code=False) + + super(Debian, self).install_virtualenv() + + +class Fedora(Distro): + """This covers all Fedora-based distributions. + + Includes: Fedora, RHEL, CentOS, Scientific Linux""" + + def check_pkg(self, pkg): + return run_command_with_code(['rpm', '-q', pkg], + check_exit_code=False)[1] == 0 + + def yum_install(self, pkg, **kwargs): + run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs) + + def apply_patch(self, originalfile, patchfile): + run_command(['patch', originalfile, patchfile]) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if not self.check_pkg('python-virtualenv'): + self.yum_install('python-virtualenv', check_exit_code=False) + + super(Fedora, self).install_virtualenv() + + +def get_distro(): + if os.path.exists('/etc/fedora-release') or \ + os.path.exists('/etc/redhat-release'): + return Fedora() + elif os.path.exists('/etc/debian_version'): + return Debian() + else: + return Distro() + + +def check_dependencies(): + get_distro().install_virtualenv() + + +def create_virtualenv(venv=VENV, no_site_packages=True): + """Creates the virtual environment and installs PIP only into the + virtual environment + """ + print 'Creating venv...', + if no_site_packages: + run_command(['virtualenv', '-q', '--no-site-packages', VENV]) + else: + run_command(['virtualenv', '-q', VENV]) + print 'done.' + print 'Installing pip in virtualenv...', + if not run_command(['tools/with_venv.sh', 'easy_install', + 'pip>1.0']).strip(): + die("Failed to install pip.") + print 'done.' + + +def pip_install(*args): + run_command(['tools/with_venv.sh', + 'pip', 'install', '--upgrade'] + list(args), + redirect_output=False) + + +def install_dependencies(venv=VENV): + print 'Installing dependencies with pip (this can take a while)...' + + # First things first, make sure our venv has the latest pip and distribute. + pip_install('pip') + pip_install('distribute') + + pip_install('-r', PIP_REQUIRES) + pip_install('-r', TEST_REQUIRES) + + # Tell the virtual env how to "import openstackclient" + pthfile = os.path.join(venv, "lib", PY_VERSION, "site-packages", + "openstackclient.pth") + f = open(pthfile, 'w') + f.write("%s\n" % ROOT) + + +def post_process(): + get_distro().post_process() + + +def print_help(): + help = """ + python-openstackclient development environment setup is complete. + + python-openstackclient development uses virtualenv to track and manage Python + dependencies while in development and testing. + + To activate the python-openstackclient virtualenv for the extent of your current + shell session you can run: + + $ source .venv/bin/activate + + Or, if you prefer, you can run commands in the virtualenv on a case by case + basis by running: + + $ tools/with_venv.sh + + Also, make test will automatically use the virtualenv. + """ + print help + + +def parse_args(): + """Parse command-line arguments""" + parser = optparse.OptionParser() + parser.add_option("-n", "--no-site-packages", dest="no_site_packages", + default=False, action="store_true", + help="Do not inherit packages from global Python install") + return parser.parse_args() + + +def main(argv): + (options, args) = parse_args() + check_python_version() + check_dependencies() + create_virtualenv(no_site_packages=options.no_site_packages) + install_dependencies() + post_process() + print_help() + +if __name__ == '__main__': + main(sys.argv) diff --git a/tools/pip-requires b/tools/pip-requires index 1aa8fed1d..af2c56ad6 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -1,11 +1,6 @@ -argparse cliff -cmd2 -httplib2 keyring -prettytable -pyparsing>=1.5.6,<2.0 pycrypto +python-glanceclient>=0.5.1 python-keystoneclient>=0.2,<1.0 python-novaclient>=2 -simplejson diff --git a/tools/test-requires b/tools/test-requires index 40350f025..d96d52b60 100644 --- a/tools/test-requires +++ b/tools/test-requires @@ -1,14 +1,12 @@ distribute>=0.6.24 -coverage fixtures mock -mox nose nose-exclude -nosehtmloutput nosexcover +nosehtmloutput openstack.nose_plugin -pep8==0.6.1 +pep8==1.1 sphinx>=1.1.2 testtools>=0.9.22 diff --git a/tools/with_venv.sh b/tools/with_venv.sh index e6e44f599..c8d2940fc 100755 --- a/tools/with_venv.sh +++ b/tools/with_venv.sh @@ -1,10 +1,4 @@ #!/bin/bash - -command -v tox > /dev/null 2>&1 -if [ $? -ne 0 ]; then - echo 'This script requires "tox" to run.' - echo 'You can install it with "pip install tox".' - exit 1; -fi - -tox -evenv -- $@ +TOOLS=`dirname $0` +VENV=$TOOLS/../.venv +source $VENV/bin/activate && $@