From 6a3f66f0e92ab409c66a38ef3c8133e4bd49a11e Mon Sep 17 00:00:00 2001 From: mbasnight Date: Wed, 22 Feb 2012 19:26:52 -0600 Subject: [PATCH] Adding the venv/unit test framework stuff * run_tests stuff * gerrit stuff * test-requires for pip --- .gitignore | 2 + development/development_enviroment.sh | 6 +- reddwarf/tests/__init__.py | 115 ++++++++ reddwarf/tests/unit/__init__.py | 1 + run_tests.py | 367 ++++++++++++++++++++++++++ run_tests.sh | 168 ++++++++++++ setup.py | 100 +++++++ tools/install_venv.py | 148 +++++++++++ tools/pip-requires | 16 ++ tools/rfc.sh | 145 ++++++++++ tools/test-requires | 9 + tools/with_venv.sh | 4 + 12 files changed, 1080 insertions(+), 1 deletion(-) create mode 100644 reddwarf/tests/__init__.py create mode 100644 reddwarf/tests/unit/__init__.py create mode 100644 run_tests.py create mode 100755 run_tests.sh create mode 100644 setup.py create mode 100644 tools/install_venv.py create mode 100644 tools/pip-requires create mode 100644 tools/rfc.sh create mode 100644 tools/test-requires create mode 100644 tools/with_venv.sh diff --git a/.gitignore b/.gitignore index 5103ec26e6..1a8ae70163 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ vagrant/ *.pyc .idea reddwarf_test.sqlite +.venv +run_tests.log diff --git a/development/development_enviroment.sh b/development/development_enviroment.sh index 7610c1ce35..3fffdb1fa5 100644 --- a/development/development_enviroment.sh +++ b/development/development_enviroment.sh @@ -2,6 +2,10 @@ # 1 install nova via devstack # 2 install reddwarf via this (or eventually mod devstack) # 3 run tempest tests + +#Kind of annoying, but the lxml stuff does not work unless u have these installed +sudo apt-get install libxml2-dev libxslt-dev + cd ~ git clone git://github.com/openstack-dev/devstack.git cd devstack @@ -36,4 +40,4 @@ curl -d '{"auth":{"passwordCredentials":{"username": "reddwarf", "password": "RE # curl -H'X-Auth-Token:AUTH_TOKEN' http://0.0.0.0:8779/v0.1/$REDDWARF_TENANT/instances # Also, you should start up the api node like this -# bin/reddwarf-api-os-database --flagfile=etc/nova/nova.conf.template +# bin/reddwarf-server --config-file=etc/reddwarf/reddwarf.conf.sample diff --git a/reddwarf/tests/__init__.py b/reddwarf/tests/__init__.py new file mode 100644 index 0000000000..9f43473172 --- /dev/null +++ b/reddwarf/tests/__init__.py @@ -0,0 +1,115 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# See http://code.google.com/p/python-nose/issues/detail?id=373 +# The code below enables nosetests to work with i18n _() blocks + + + +import __builtin__ +setattr(__builtin__, '_', lambda x: x) + +import os +import unittest +import urlparse + +import mox + +from reddwarf.db import db_api +from reddwarf.common import utils + + +def reddwarf_root_path(): + return os.path.join(os.path.dirname(__file__), "..", "..") + + +def reddwarf_bin_path(filename="."): + return os.path.join(reddwarf_root_path(), "bin", filename) + + +def reddwarf_etc_path(filename="."): + return os.path.join(reddwarf_root_path(), "etc", "reddwarf", filename) + + +def test_config_file(): + return reddwarf_etc_path("reddwarf.conf.sample") + + +class BaseTest(unittest.TestCase): + + def setUp(self): + #maxDiff=None ensures diff output of assert methods are not truncated + self.maxDiff = None + + self.mock = mox.Mox() + db_api.clean_db() + super(BaseTest, self).setUp() + + def tearDown(self): + self.mock.UnsetStubs() + self.mock.VerifyAll() + super(BaseTest, self).tearDown() + + def assertRaisesExcMessage(self, exception, message, + func, *args, **kwargs): + """This is similar to assertRaisesRegexp in python 2.7""" + + try: + func(*args, **kwargs) + self.fail("Expected {0} to raise {1}".format(func, + repr(exception))) + except exception as error: + self.assertIn(message, str(error)) + + def assertIn(self, expected, actual): + """This is similar to assertIn in python 2.7""" + self.assertTrue(expected in actual, + "{0} does not contain {1}".format(repr(actual), repr(expected))) + + def assertNotIn(self, expected, actual): + self.assertFalse(expected in actual, + "{0} does contains {1}".format(repr(actual), repr(expected))) + + def assertIsNone(self, actual): + """This is similar to assertIsNone in python 2.7""" + self.assertEqual(actual, None) + + def assertIsNotNone(self, actual): + """This is similar to assertIsNotNone in python 2.7""" + self.assertNotEqual(actual, None) + + def assertItemsEqual(self, expected, actual): + self.assertEqual(sorted(expected), sorted(actual)) + + def assertModelsEqual(self, expected, actual): + self.assertEqual(sorted(expected, key=lambda model: model.id), + sorted(actual, key=lambda model: model.id)) + + def assertUrlEqual(self, expected, actual): + self.assertEqual(expected.partition("?")[0], actual.partition("?")[0]) + + #params ordering might be different in the urls + self.assertEqual(urlparse.parse_qs(expected.partition("?")[2]), + urlparse.parse_qs(actual.partition("?")[2])) + + def assertErrorResponse(self, response, error_type, expected_error): + self.assertEqual(response.status_int, error_type().code) + self.assertIn(expected_error, response.body) + + def setup_uuid_with(self, fake_uuid): + self.mock.StubOutWithMock(utils, "generate_uuid") + utils.generate_uuid().MultipleTimes().AndReturn(fake_uuid) diff --git a/reddwarf/tests/unit/__init__.py b/reddwarf/tests/unit/__init__.py new file mode 100644 index 0000000000..0519dd3ece --- /dev/null +++ b/reddwarf/tests/unit/__init__.py @@ -0,0 +1 @@ +__author__ = 'mbasnight' diff --git a/run_tests.py b/run_tests.py new file mode 100644 index 0000000000..4da9ee1e43 --- /dev/null +++ b/run_tests.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Colorizer Code is borrowed from Twisted: +# Copyright (c) 2001-2010 Twisted Matrix Laboratories. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Unittest runner for Nova. + +To run all tests + python run_tests.py + +To run a single test: + python run_tests.py test_compute:ComputeTestCase.test_run_terminate + +To run a single test module: + python run_tests.py test_compute + + or + + python run_tests.py api.test_wsgi + +""" + +import gettext +import heapq +import logging +import os +import unittest +import sys +import time + +gettext.install('reddwarf', unicode=1) + +from nose import config +from nose import core +from nose import result + + +class _AnsiColorizer(object): + """ + A colorizer is an object that loosely wraps around a stream, allowing + callers to write text to the stream in a particular color. + + Colorizer classes must implement C{supported()} and C{write(text, color)}. + """ + _colors = dict(black=30, red=31, green=32, yellow=33, + blue=34, magenta=35, cyan=36, white=37) + + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + """ + A class method that returns True if the current platform supports + coloring terminal output using this method. Returns False otherwise. + """ + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + except ImportError: + return False + else: + try: + try: + return curses.tigetnum("colors") > 2 + except curses.error: + curses.setupterm() + return curses.tigetnum("colors") > 2 + except: + raise + # guess false in case of error + return False + supported = classmethod(supported) + + def write(self, text, color): + """ + Write the given text to the stream in the given color. + + @param text: Text to be written to the stream. + + @param color: A string label for a color. e.g. 'red', 'white'. + """ + color = self._colors[color] + self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) + + +class _Win32Colorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + from win32console import GetStdHandle, STD_OUT_HANDLE,\ + FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN,\ + FOREGROUND_INTENSITY + red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN, + FOREGROUND_BLUE, FOREGROUND_INTENSITY) + self.stream = stream + self.screenBuffer = GetStdHandle(STD_OUT_HANDLE) + self._colors = { + 'normal': red | green | blue, + 'red': red | bold, + 'green': green | bold, + 'blue': blue | bold, + 'yellow': red | green | bold, + 'magenta': red | blue | bold, + 'cyan': green | blue | bold, + 'white': red | green | blue | bold + } + + def supported(cls, stream=sys.stdout): + try: + import win32console + screenBuffer = win32console.GetStdHandle( + win32console.STD_OUT_HANDLE) + except ImportError: + return False + import pywintypes + try: + screenBuffer.SetConsoleTextAttribute( + win32console.FOREGROUND_RED | + win32console.FOREGROUND_GREEN | + win32console.FOREGROUND_BLUE) + except pywintypes.error: + return False + else: + return True + supported = classmethod(supported) + + def write(self, text, color): + color = self._colors[color] + self.screenBuffer.SetConsoleTextAttribute(color) + self.stream.write(text) + self.screenBuffer.SetConsoleTextAttribute(self._colors['normal']) + + +class _NullColorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + return True + supported = classmethod(supported) + + def write(self, text, color): + self.stream.write(text) + + +def get_elapsed_time_color(elapsed_time): + if elapsed_time > 1.0: + return 'red' + elif elapsed_time > 0.25: + return 'yellow' + else: + return 'green' + + +class ReddwarfTestResult(result.TextTestResult): + def __init__(self, *args, **kw): + self.show_elapsed = kw.pop('show_elapsed') + result.TextTestResult.__init__(self, *args, **kw) + self.num_slow_tests = 5 + self.slow_tests = [] # this is a fixed-sized heap + self._last_case = None + self.colorizer = None + # NOTE(vish): reset stdout for the terminal check + stdout = sys.stdout + sys.stdout = sys.__stdout__ + for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: + if colorizer.supported(): + self.colorizer = colorizer(self.stream) + break + sys.stdout = stdout + + # NOTE(lorinh): Initialize start_time in case a sqlalchemy-migrate + # error results in it failing to be initialized later. Otherwise, + # _handleElapsedTime will fail, causing the wrong error message to + # be outputted. + self.start_time = time.time() + + def getDescription(self, test): + return str(test) + + def _handleElapsedTime(self, test): + self.elapsed_time = time.time() - self.start_time + item = (self.elapsed_time, test) + # Record only the n-slowest tests using heap + if len(self.slow_tests) >= self.num_slow_tests: + heapq.heappushpop(self.slow_tests, item) + else: + heapq.heappush(self.slow_tests, item) + + def _writeElapsedTime(self, test): + color = get_elapsed_time_color(self.elapsed_time) + self.colorizer.write(" %.2f" % self.elapsed_time, color) + + def _writeResult(self, test, long_result, color, short_result, success): + if self.showAll: + self.colorizer.write(long_result, color) + if self.show_elapsed and success: + self._writeElapsedTime(test) + self.stream.writeln() + elif self.dots: + self.stream.write(short_result) + self.stream.flush() + + # NOTE(vish): copied from unittest with edit to add color + def addSuccess(self, test): + unittest.TestResult.addSuccess(self, test) + self._handleElapsedTime(test) + self._writeResult(test, 'OK', 'green', '.', True) + + # NOTE(vish): copied from unittest with edit to add color + def addFailure(self, test, err): + unittest.TestResult.addFailure(self, test, err) + self._handleElapsedTime(test) + self._writeResult(test, 'FAIL', 'red', 'F', False) + + # NOTE(vish): copied from nose with edit to add color + def addError(self, test, err): + """Overrides normal addError to add support for + errorClasses. If the exception is a registered class, the + error will be added to the list for that class, not errors. + """ + self._handleElapsedTime(test) + stream = getattr(self, 'stream', None) + ec, ev, tb = err + try: + exc_info = self._exc_info_to_string(err, test) + except TypeError: + # 2.3 compat + exc_info = self._exc_info_to_string(err) + for cls, (storage, label, isfail) in self.errorClasses.items(): + if result.isclass(ec) and issubclass(ec, cls): + if isfail: + test.passed = False + storage.append((test, exc_info)) + # Might get patched into a streamless result + if stream is not None: + if self.showAll: + message = [label] + detail = result._exception_detail(err[1]) + if detail: + message.append(detail) + stream.writeln(": ".join(message)) + elif self.dots: + stream.write(label[:1]) + return + self.errors.append((test, exc_info)) + test.passed = False + if stream is not None: + self._writeResult(test, 'ERROR', 'red', 'E', False) + + def startTest(self, test): + unittest.TestResult.startTest(self, test) + self.start_time = time.time() + current_case = test.test.__class__.__name__ + + if self.showAll: + if current_case != self._last_case: + self.stream.writeln(current_case) + self._last_case = current_case + + self.stream.write( + ' %s' % str(test.test._testMethodName).ljust(60)) + self.stream.flush() + + +class ReddwarfTestRunner(core.TextTestRunner): + def __init__(self, *args, **kwargs): + self.show_elapsed = kwargs.pop('show_elapsed') + core.TextTestRunner.__init__(self, *args, **kwargs) + + def _makeResult(self): + return ReddwarfTestResult(self.stream, + self.descriptions, + self.verbosity, + self.config, + show_elapsed=self.show_elapsed) + + def _writeSlowTests(self, result_): + # Pare out 'fast' tests + slow_tests = [item for item in result_.slow_tests + if get_elapsed_time_color(item[0]) != 'green'] + if slow_tests: + slow_total_time = sum(item[0] for item in slow_tests) + self.stream.writeln("Slowest %i tests took %.2f secs:" + % (len(slow_tests), slow_total_time)) + for elapsed_time, test in sorted(slow_tests, reverse=True): + time_str = "%.2f" % elapsed_time + self.stream.writeln(" %s %s" % (time_str.ljust(10), test)) + + def run(self, test): + result_ = core.TextTestRunner.run(self, test) + if self.show_elapsed: + self._writeSlowTests(result_) + return result_ + + +if __name__ == '__main__': + logger = logging.getLogger() + hdlr = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + hdlr.setFormatter(formatter) + logger.addHandler(hdlr) + logger.setLevel(logging.DEBUG) + # If any argument looks like a test name but doesn't have "reddwarf.tests" in + # front of it, automatically add that so we don't have to type as much + show_elapsed = True + argv = [] + for x in sys.argv: + if x.startswith('test_'): + argv.append('reddwarf.tests.%s' % x) + elif x.startswith('--hide-elapsed'): + show_elapsed = False + else: + argv.append(x) + + testdir = os.path.abspath(os.path.join("reddwarf", "tests")) + c = config.Config(stream=sys.stdout, + env=os.environ, + verbosity=3, + workingDir=testdir, + plugins=core.DefaultPluginManager()) + + runner = ReddwarfTestRunner(stream=c.stream, + verbosity=c.verbosity, + config=c, + show_elapsed=show_elapsed) + sys.exit(not core.run(config=c, testRunner=runner, argv=argv)) diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 0000000000..cc330b9d0f --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +set -eu + +function usage { + echo "Usage: $0 [OPTION]..." + echo "Run Reddwarf's test suite(s)" + echo "" + echo " -V, --virtual-env Always use virtualenv. Install automatically if not present" + echo " -N, --no-virtual-env Don't use virtualenv. Run tests in local environment" + echo " -r, --recreate-db Recreate the test database (deprecated, as this is now the default)." + echo " -n, --no-recreate-db Don't recreate the test database." + echo " -x, --stop Stop running tests after the first error or failure." + echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added." + echo " -p, --pep8 Just run pep8" + echo " -P, --no-pep8 Don't run pep8" + echo " -c, --coverage Generate coverage report" + echo " -h, --help Print this usage message" + echo " --hide-elapsed Don't print the elapsed time for each test along with slow test list" + echo "" + echo "Note: with no options specified, the script will try to run the tests in a virtual environment," + echo " If no virtualenv is found, the script will ask if you would like to create one. If you " + echo " prefer to run tests NOT in a virtual environment, simply pass the -N option." + exit +} + +function process_option { + case "$1" in + -h|--help) usage;; + -V|--virtual-env) always_venv=1; never_venv=0;; + -N|--no-virtual-env) always_venv=0; never_venv=1;; + -r|--recreate-db) recreate_db=1;; + -n|--no-recreate-db) recreate_db=0;; + -f|--force) force=1;; + -p|--pep8) just_pep8=1;; + -P|--no-pep8) no_pep8=1;; + -c|--coverage) coverage=1;; + -*) noseopts="$noseopts $1";; + *) noseargs="$noseargs $1" + esac +} + +venv=.venv +with_venv=tools/with_venv.sh +always_venv=0 +never_venv=0 +force=0 +noseargs= +noseopts= +wrapper="" +just_pep8=0 +no_pep8=0 +coverage=0 +recreate_db=1 + +for arg in "$@"; do + process_option $arg +done + +# If enabled, tell nose to collect coverage data +if [ $coverage -eq 1 ]; then + noseopts="$noseopts --with-coverage --cover-package=reddwarf" +fi + +function run_tests { + # Just run the test suites in current environment + ${wrapper} $NOSETESTS 2> run_tests.log + # If we get some short import error right away, print the error log directly + RESULT=$? + if [ "$RESULT" -ne "0" ]; + then + ERRSIZE=`wc -l run_tests.log | awk '{print \$1}'` + if [ "$ERRSIZE" -lt "40" ]; + then + cat run_tests.log + fi + fi + return $RESULT +} + +function run_pep8 { + echo "Running pep8 ..." + # Opt-out files from pep8 + ignore_scripts="*.sh:*reddwarf-debug:*clean-vlans" + ignore_files="*eventlet-patch:*pip-requires:*test-requires" + GLOBIGNORE="$ignore_scripts:$ignore_files" + srcfiles=`find bin -type f ! -name "reddwarf.conf*"` + srcfiles+=" `find tools/*`" + srcfiles+=" setup.py bin reddwarf" + # Just run PEP8 in current environment + # + # NOTE(sirp): W602 (deprecated 3-arg raise) is being ignored for the + # following reasons: + # + # 1. It's needed to preserve traceback information when re-raising + # exceptions; this is needed b/c Eventlet will clear exceptions when + # switching contexts. + # + # 2. There doesn't appear to be an alternative, "pep8-tool" compatible way of doing this + # in Python 2 (in Python 3 `with_traceback` could be used). + # + # 3. Can find no corroborating evidence that this is deprecated in Python 2 + # other than what the PEP8 tool claims. It is deprecated in Python 3, so, + # perhaps the mistake was thinking that the deprecation applied to Python 2 + # as well. + ${wrapper} pep8 --repeat --show-pep8 --show-source \ + --ignore=E202,W602 \ + --exclude=vcsversion.py ${srcfiles} +} + +NOSETESTS="python run_tests.py $noseopts $noseargs" + +if [ $never_venv -eq 0 ] +then + # Remove the virtual environment if --force used + if [ $force -eq 1 ]; then + echo "Cleaning virtualenv..." + rm -rf ${venv} + fi + if [ -e ${venv} ]; then + wrapper="${with_venv}" + else + if [ $always_venv -eq 1 ]; then + # Automatically install the virtualenv + python tools/install_venv.py + wrapper="${with_venv}" + else + echo -e "No virtual environment found...create one? (Y/n) \c" + read use_ve + if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then + # Install the virtualenv and run the test suite in it + python tools/install_venv.py + wrapper=${with_venv} + fi + fi + fi +fi + +# Delete old coverage data from previous runs +if [ $coverage -eq 1 ]; then + ${wrapper} coverage erase +fi + +if [ $just_pep8 -eq 1 ]; then + run_pep8 + exit +fi + +if [ $recreate_db -eq 1 ]; then + rm -f tests.sqlite +fi + +run_tests + +# NOTE(sirp): we only want to run pep8 when we're running the full-test suite, +# not when we're running tests individually. To handle this, we need to +# distinguish between options (noseopts), which begin with a '-', and +# arguments (noseargs). +if [ -z "$noseargs" ]; then + if [ $no_pep8 -eq 0 ]; then + run_pep8 + fi +fi + +if [ $coverage -eq 1 ]; then + echo "Generating coverage report in covhtml/" + ${wrapper} coverage html -d covhtml -i +fi diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..3133996bb3 --- /dev/null +++ b/setup.py @@ -0,0 +1,100 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import gettext +import os +import subprocess + +from setuptools import find_packages +from setuptools.command.sdist import sdist +from setuptools import setup + +gettext.install('reddwarf', unicode=1) + +from reddwarf.openstack.common.setup import parse_requirements +from reddwarf.openstack.common.setup import parse_dependency_links +from reddwarf.openstack.common.setup import write_requirements +from reddwarf.openstack.common.setup import write_vcsversion +from reddwarf.openstack.common.setup import write_git_changelog + +from reddwarf import version + + +class local_sdist(sdist): + """Customized sdist hook - builds the ChangeLog file from VC first""" + def run(self): + write_git_changelog() + sdist.run(self) +cmdclass = {'sdist': local_sdist} + + +try: + from sphinx.setup_command import BuildDoc + + class local_BuildDoc(BuildDoc): + def run(self): + for builder in ['html', 'man']: + self.builder = builder + self.finalize_options() + BuildDoc.run(self) + cmdclass['build_sphinx'] = local_BuildDoc + +except: + pass + + +try: + from babel.messages import frontend as babel + cmdclass['compile_catalog'] = babel.compile_catalog + cmdclass['extract_messages'] = babel.extract_messages + cmdclass['init_catalog'] = babel.init_catalog + cmdclass['update_catalog'] = babel.update_catalog +except: + pass + +requires = parse_requirements() +depend_links = parse_dependency_links() + +write_requirements() +write_vcsversion('reddwarf/vcsversion.py') + +setup(name='reddwarf', + version=version.canonical_version_string(), + description='PaaS services for Openstack', + author='OpenStack', + author_email='openstack@lists.launchpad.net', + url='http://www.openstack.org/', + cmdclass=cmdclass, + packages=find_packages(exclude=['bin']), + include_package_data=True, + install_requires=requires, + dependency_links=depend_links, + test_suite='nose.collector', + classifiers=[ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 2.6', + 'Environment :: No Input/Output (Daemon)', + ], + scripts=['bin/reddwarf-server', + 'bin/reddwarf-manage', + ], + py_modules=[], + namespace_packages=['reddwarf'], +) diff --git a/tools/install_venv.py b/tools/install_venv.py new file mode 100644 index 0000000000..ff644d5346 --- /dev/null +++ b/tools/install_venv.py @@ -0,0 +1,148 @@ + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2010 OpenStack, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Installation script for Reddwarf's development virtualenv +""" + +import os +import subprocess +import sys + + +ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +VENV = os.path.join(ROOT, '.venv') +PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires') +TEST_REQUIRES = os.path.join(ROOT, 'tools', 'test-requires') +PY_VERSION = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) + + +def die(message, *args): + print >> sys.stderr, message % args + sys.exit(1) + + +def check_python_version(): + if sys.version_info < (2, 6): + die("Need Python Version >= 2.6") + + +def run_command(cmd, redirect_output=True, check_exit_code=True): + """ + Runs a command in an out-of-process shell, returning the + output of that command. Working directory is ROOT. + """ + if redirect_output: + stdout = subprocess.PIPE + else: + stdout = None + + proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout) + output = proc.communicate()[0] + if check_exit_code and proc.returncode != 0: + die('Command "%s" failed.\n%s', ' '.join(cmd), output) + return output + + +HAS_EASY_INSTALL = bool(run_command(['which', 'easy_install'], + check_exit_code=False).strip()) +HAS_VIRTUALENV = bool(run_command(['which', 'virtualenv'], + check_exit_code=False).strip()) + + +def check_dependencies(): + """Make sure virtualenv is in the path.""" + + if not HAS_VIRTUALENV: + print 'not found.' + # Try installing it via easy_install... + if HAS_EASY_INSTALL: + print 'Installing virtualenv via easy_install...', + if not (run_command(['which', 'easy_install']) and + run_command(['easy_install', 'virtualenv'])): + die('ERROR: virtualenv not found.\n\Reddwarf development' + ' requires virtualenv, please install it using your' + ' favorite package management tool') + print 'done.' + print 'done.' + + +def create_virtualenv(venv=VENV): + """Creates the virtual environment and installs PIP only into the + virtual environment + """ + print 'Creating venv...', + run_command(['virtualenv', '-q', '--no-site-packages', VENV]) + print 'done.' + print 'Installing pip in virtualenv...', + if not run_command(['tools/with_venv.sh', 'easy_install', 'pip']).strip(): + die("Failed to install pip.") + print 'done.' + + +def install_dependencies(venv=VENV): + print 'Installing dependencies with pip (this can take a while)...' + # Install greenlet by hand - just listing it in the requires file does not + # get it in stalled in the right order + run_command(['tools/with_venv.sh', 'pip', 'install', '-E', venv, + 'greenlet'], redirect_output=False) + for requires in (PIP_REQUIRES, TEST_REQUIRES): + run_command(['tools/with_venv.sh', 'pip', 'install', '-E', venv, '-r', + requires], redirect_output=False) + + # Tell the virtual env how to "import reddwarf" + pthfile = os.path.join(venv, "lib", PY_VERSION, "site-packages", + "reddwarf.pth") + f = open(pthfile, 'w') + f.write("%s\n" % ROOT) + + +def print_help(): + help = """ + Reddwarf development environment setup is complete. + + Reddwarf development uses virtualenv to track and manage Python + dependencies while in development and testing. + + To activate the Reddwarf virtualenv for the extent of your current shell + session you can run: + + $ source .venv/bin/activate + + Or, if you prefer, you can run commands in the virtualenv on a case by case + basis by running: + + $ tools/with_venv.sh + + Also, make test will automatically use the virtualenv. + """ + print help + + +def main(argv): + check_python_version() + check_dependencies() + create_virtualenv() + install_dependencies() + print_help() + +if __name__ == '__main__': + main(sys.argv) diff --git a/tools/pip-requires b/tools/pip-requires new file mode 100644 index 0000000000..749f235c2e --- /dev/null +++ b/tools/pip-requires @@ -0,0 +1,16 @@ +SQLAlchemy +eventlet +kombu==1.5.1 +routes +WebOb +mox +PasteDeploy +paste +sqlalchemy-migrate +netaddr +sphinx +webtest +factory_boy +httplib2 +lxml +python-novaclient diff --git a/tools/rfc.sh b/tools/rfc.sh new file mode 100644 index 0000000000..ddcc22c5ef --- /dev/null +++ b/tools/rfc.sh @@ -0,0 +1,145 @@ +#!/bin/sh -e +# Copyright (c) 2010-2011 Gluster, Inc. +# This initial version of this file was taken from the source tree +# of GlusterFS. It was not directly attributed, but is assumed to be +# Copyright (c) 2010-2011 Gluster, Inc and release GPLv3 +# Subsequent modifications are Copyright (c) 2011 OpenStack, LLC. +# +# GlusterFS is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3 of the License, +# or (at your option) any later version. +# +# GlusterFS is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# . + + +branch="master"; + +set_hooks_commit_msg() +{ + top_dir=`git rev-parse --show-toplevel` + f="${top_dir}/.git/hooks/commit-msg"; + u="https://review.openstack.org/tools/hooks/commit-msg"; + + if [ -x "$f" ]; then + return; + fi + + curl -o $f $u || wget -O $f $u; + + chmod +x $f; + + GIT_EDITOR=true git commit --amend +} + +add_remote() +{ + username=$1 + project=$2 + + echo "No remote set, testing ssh://$username@review.openstack.org:29418" + if project_list=`ssh -p29418 -o StrictHostKeyChecking=no $username@review.openstack.org gerrit ls-projects 2>/dev/null` + then + echo "$username@review.openstack.org:29418 worked." + if echo $project_list | grep -w "${project%.git}" >/dev/null + then + echo "Creating a git remote called gerrit that maps to:" + echo " ssh://$username@review.openstack.org:29418/$project" + git remote add gerrit ssh://$username@review.openstack.org:29418/$project + else + echo "The current project name, ${project%.git}, is not a known project." + echo "Please either reclone from github/gerrit or create a" + echo "remote named gerrit that points to the intended project." + return 1 + fi + + return 0 + fi + return 1 +} + +check_remote() +{ + if ! git remote | grep gerrit >/dev/null 2>&1 + then + origin_project=`git remote show origin | grep 'Fetch URL' | perl -nle '@fields = split(m|[:/]|); $len = $#fields; print $fields[$len-1], "/", $fields[$len];'` + if add_remote $USERNAME $origin_project + then + return 0 + else + echo "Your local name doesn't work on Gerrit." + echo -n "Enter Gerrit username (same as launchpad): " + read gerrit_user + if add_remote $gerrit_user $origin_project + then + return 0 + else + echo "Can't infer where gerrit is - please set a remote named" + echo "gerrit manually and then try again." + echo + echo "For more information, please see:" + echo "\thttp://wiki.openstack.org/GerritWorkflow" + exit 1 + fi + fi + fi +} + +rebase_changes() +{ + git fetch; + + GIT_EDITOR=true git rebase -i origin/$branch || exit $?; +} + + +assert_diverge() +{ + if ! git diff origin/$branch..HEAD | grep -q . + then + echo "No changes between the current branch and origin/$branch." + exit 1 + fi +} + + +main() +{ + set_hooks_commit_msg; + + check_remote; + + rebase_changes; + + assert_diverge; + + bug=$(git show --format='%s %b' | perl -nle 'if (/\b([Bb]ug|[Ll][Pp])\s*[#:]?\s*(\d+)/) {print "$2"; exit}') + + bp=$(git show --format='%s %b' | perl -nle 'if (/\b([Bb]lue[Pp]rint|[Bb][Pp])\s*[#:]?\s*([0-9a-zA-Z-_]+)/) {print "$2"; exit}') + + if [ "$DRY_RUN" = 1 ]; then + drier='echo -e Please use the following command to send your commits to review:\n\n' + else + drier= + fi + + local_branch=`git branch | grep -Ei "\* (.*)" | cut -f2 -d' '` + if [ -z "$bug" ]; then + if [ -z "$bp" ]; then + $drier git push gerrit HEAD:refs/for/$branch/$local_branch; + else + $drier git push gerrit HEAD:refs/for/$branch/bp/$bp; + fi + else + $drier git push gerrit HEAD:refs/for/$branch/bug/$bug; + fi +} + +main "$@" diff --git a/tools/test-requires b/tools/test-requires new file mode 100644 index 0000000000..71009cae8d --- /dev/null +++ b/tools/test-requires @@ -0,0 +1,9 @@ +# Packages needed for dev testing +distribute>=0.6.24 + +coverage +nose +nosexcover +openstack.nose_plugin +pep8 +pylint diff --git a/tools/with_venv.sh b/tools/with_venv.sh new file mode 100644 index 0000000000..c8d2940fc7 --- /dev/null +++ b/tools/with_venv.sh @@ -0,0 +1,4 @@ +#!/bin/bash +TOOLS=`dirname $0` +VENV=$TOOLS/../.venv +source $VENV/bin/activate && $@