Gets Keystone a bit more inline with the way that

other OpenStack projects run tests. Basically, adds the standard
run_tests.sh script, modifies the run_tests.py script to do the
following:

a) Correctly create a test configuration file that is passed
   to keystone-manage and keystone. This allows you to test
   keystone on a server that already has keystone running on it
b) Some DRY cleanup around the removal of database files that
   get created in the course of testing
c) Creates a virtualenv automatically with the -V (-f) flag,
   which reads the tools/pip-requires file and constructs a virtualenv
   with all dependencies installed

Change-Id: I8defa5956a7f23258936d04a11655d36a71226ec

.gitignore

@@ -16,3 +16,6 @@ pidfile
 doc/build/
 build/
 doc/guide/target
+.keystone-venv/
+keystone.egg-info/
+run_tests.err.log

README.md

@@ -97,16 +97,10 @@ You may need to prefix your `pip install` commands with `sudo`, depending on you
 # Show dependencies
 $ cat tools/pip-requires
 
-# Install dependencies
+# Install dependencies (for production, testing, and development)
 $ pip install -r tools/pip-requires
 
-# Install test dependencies
-$ pip install -r tools/pip-requires-testing
-
-# Install development dependencies
-$ pip install -r tools/pip-requires-development
-
-#Install Memcache (If memcache is one of the backends enabled)
+# Optional: Install Memcache (if enabled as a backend)
 Refer #(http://memcached.org/)
 </pre>
 
@@ -126,6 +120,7 @@ Starting the admin server only (exposes the Admin API):
 
 By default, configuration parameters (such as the IP and port binding for each service) are parsed from `etc/keystone.conf`.
 
+
 ## Running Tests
 
 Before running tests, ensure you have installed the testing dependencies as described in the Dependencies section above.
@@ -134,7 +129,8 @@ To run the test suite in a single command:
 
     $ python keystone/test/run_tests.py
 
-#### Test data
+
+#### Sample data
 A set of sample data can be added by running a shell script:
 
     $ ./bin/sampledata.sh
@@ -143,6 +139,7 @@ The script calls `keystone-manage` to create the sample data.
 
 After starting keystone or running `keystone-manage` a `keystone.db` sqlite database should be created in the keystone folder.
 
+
 #### Demo
 To run client demo (with all auth middleware running locally on sample service):
 
@@ -150,20 +147,6 @@ To run client demo (with all auth middleware running locally on sample service):
     $ python examples/echo/echo_client.py
 
 
-#### Unit Tests
-There are 10 groups of tests. They can be run individually or as an entire colection. To run the entire test suite run:
-
-    $ python keystone/test/unit/test_keystone.py
-
-A test can also be run individually, e.g.:
-
-    $ python keystone/test/unit/test_token.py
-
-For more on unit testing please refer to:
-
-    $ python keystone/test/unit/test_keystone.py --help
-
-
 #### API Validation
 To perform contract validation and load testing, use SoapUI (for now).
 
@@ -394,17 +377,18 @@ We could potentially integrate with those:
 #### On a Mac
 
 Using macports:
-sudo port install openldap
 
-Looks like python-ldap needs recompiling to work. So:
-download it from here: http://pypi.python.org/pypi/python-ldap/2.4.1
-unpack it and go to the unpacked directory
+	sudo port install openldap
 
-edit setup.cfg (set lines below):
+It appears the package `python-ldap` needs to be recompiled to work. So,
+download it from: http://pypi.python.org/pypi/python-ldap/2.4.1
 
-  library_dirs = /opt/local/lib
-  include_dirs = /opt/local/include /usr/include/sasl
+After unpacking, edit `setup.cfg` as shown below:
 
-then run:
-python setup.py build
-sudo python setup.py install
+	library_dirs = /opt/local/lib
+	include_dirs = /opt/local/include /usr/include/sasl
+
+Then, run:
+
+	python setup.py build
+	sudo python setup.py install

keystone/identity.wadl

@@ -1 +0,0 @@
-../doc/guide/src/docbkx/identity.wadl

keystone/test/etc/ldap.conf.template

@@ -2,7 +2,7 @@
 verbose = False
 debug = False
 default_store = sqlite
-log_file = keystone.ldap.log
+log_file = %(test_dir)s/keystone.ldap.log
 backends = keystone.backends.sqlalchemy,keystone.backends.ldap
 service-header-mappings = {
 	'nova' : 'X-Server-Management-Url',
@@ -13,14 +13,15 @@ service_port = 5000
 admin_host = 0.0.0.0
 admin_port = 5001
 keystone-admin-role = Admin
+keystone-service-admin-role = KeystoneServiceAdmin
 
 [keystone.backends.sqlalchemy]
-sql_connection = sqlite:///keystone.db
+sql_connection = sqlite:///%(test_dir)s/keystone.db
 sql_idle_timeout = 30
-backend_entities = ['Endpoints', 'Credentials',  'EndpointTemplates', 'Token']
+backend_entities = ['Endpoints', 'Credentials',  'EndpointTemplates', 'Token', 'Service']
 
 [keystone.backends.ldap]
-ldap_url = fake://ldap.db
+ldap_url = fake://%(test_dir)s/ldap.db
 ldap_user = cn=Admin
 ldap_password = password
 backend_entities = ['Tenant', 'User', 'UserRoleAssociation', 'Role']

keystone/test/etc/memcache.conf.template

@@ -2,7 +2,7 @@
 verbose = False
 debug = False
 default_store = sqlite
-log_file = keystone.memcache.log
+log_file = %(test_dir)s/keystone.memcache.log
 backends = keystone.backends.sqlalchemy,keystone.backends.memcache
 service-header-mappings = {
 	'nova' : 'X-Server-Management-Url',
@@ -13,11 +13,12 @@ service_port = 5000
 admin_host = 0.0.0.0
 admin_port = 5001
 keystone-admin-role = Admin
+keystone-service-admin-role = KeystoneServiceAdmin
 
 [keystone.backends.sqlalchemy]
-sql_connection = sqlite:///keystone.db
+sql_connection = sqlite:///%(test_dir)s/keystone.db
 sql_idle_timeout = 30
-backend_entities = ['Endpoints', 'Credentials',  'EndpointTemplates', 'Tenant', 'User', 'UserRoleAssociation', 'Role']
+backend_entities = ['Endpoints', 'Credentials',  'EndpointTemplates', 'Tenant', 'User', 'UserRoleAssociation', 'Role', 'Service']
 
 [keystone.backends.memcache]
 memcache_hosts = 127.0.0.1:11211

keystone/test/etc/sql.conf.template

@@ -2,7 +2,7 @@
 verbose = False
 debug = False
 default_store = sqlite
-log_file = keystone.sql.log
+log_file = %(test_dir)s/keystone.sql.log
 backends = keystone.backends.sqlalchemy
 service-header-mappings = {
 	'nova' : 'X-Server-Management-Url',
@@ -13,11 +13,12 @@ service_port = 5000
 admin_host = 0.0.0.0
 admin_port = 5001
 keystone-admin-role = Admin
+keystone-service-admin-role = KeystoneServiceAdmin
 
 [keystone.backends.sqlalchemy]
-sql_connection = sqlite:///keystone.db
+sql_connection = sqlite:///%(test_dir)s/keystone.db
 sql_idle_timeout = 30
-backend_entities = ['Endpoints', 'Credentials',  'EndpointTemplates', 'Tenant', 'User', 'UserRoleAssociation', 'Role', 'Token']
+backend_entities = ['Endpoints', 'Credentials',  'EndpointTemplates', 'Tenant', 'User', 'UserRoleAssociation', 'Role', 'Token', 'Service']
 
 [pipeline:admin]
 pipeline =

keystone/test/run_tests.py

@@ -2,54 +2,112 @@
 import os
 import sys
 import subprocess
+import tempfile
 import time
 
-TEST_DIR = os.path.dirname(__file__)
+TEST_DIR = os.path.abspath(os.path.dirname(__file__))
 
 CONFIG_FILES = (
-    'keystone.sql.conf',
-    'keystone.memcache.conf',
-    'keystone.ldap.conf')
+    'sql.conf.template',
+    #'memcache.conf.template',
+    'ldap.conf.template')
 
-TEMP_FILES = (
+TEST_FILES = (
     'keystone.db',
     'keystone.token.db',
     'ldap.db',
     'ldap.db.db')
 
 
-def delete_temp_files():
-    """Quietly deletes any temp files in the test directory"""
-    for path in TEMP_FILES:
-        subprocess.call(['rm', '-f', os.path.join(TEST_DIR, path)])
+def execute(cmd, raise_error=True):
+    """
+    Executes a command in a subprocess. Returns a tuple
+    of (exitcode, out, err), where out is the string output
+    from stdout and err is the string output from stderr when
+    executing the command.
+
+    :param cmd: Command string to execute
+    :param raise_error: If returncode is not 0 (success), then
+                        raise a RuntimeError? Default: True)
+    """
+
+    env = os.environ.copy()
+
+    # Make sure that we use the programs in the
+    # current source directory's bin/ directory.
+    env['PATH'] = os.path.join(os.getcwd(), 'bin') + ':' + env['PATH']
+    process = subprocess.Popen(cmd,
+                               shell=True,
+                               stdin=subprocess.PIPE,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE,
+                               env=env)
+    result = process.communicate()
+    (out, err) = result
+    exitcode = process.returncode
+    if process.returncode != 0 and raise_error:
+        msg = "Command %(cmd)s did not succeed. Returned an exit "\
+              "code of %(exitcode)d."\
+              "\n\nSTDOUT: %(out)s"\
+              "\n\nSTDERR: %(err)s" % locals()
+        raise RuntimeError(msg)
+    return exitcode, out, err
+
+
+def remove_test_files():
+    """Remove any test databases or files generated by previous tests."""
+    for fname in TEST_FILES:
+        fpath = os.path.join(TEST_DIR, fname)
+        if os.path.exists(fpath):
+            print "Removing test file %s" % fname
+            os.unlink(fpath)
+
+
+def construct_temp_conf_file(temp_fp, config_name):
+    """Populates a configuration template, and writes to a file pointer."""
+    template_fpath = os.path.join(TEST_DIR, 'etc', config_name)
+    conf_contents = open(template_fpath).read()
+    conf_contents = conf_contents % {'test_dir': TEST_DIR}
+    temp_fp.write(conf_contents)
+    temp_fp.flush()
+
 
 if __name__ == '__main__':
     for config in CONFIG_FILES:
-        # remove any pre-existing temp files
-        delete_temp_files()
+        print 'Using config file', CONFIG_FILES.index(config) + 1, 'of', \
+            str(len(CONFIG_FILES)) + ':', config
 
-        # populate the test database
-        subprocess.check_call([
-            os.path.join(TEST_DIR, '..', '..', 'bin', 'sampledata.sh'),
-            '-c', os.path.join(TEST_DIR, '..', '..', config)])
+        remove_test_files()
 
         try:
-            # run the keystone SERVER
-            SERVER = subprocess.Popen([
-                os.path.join(TEST_DIR, '..', '..', 'bin', 'keystone'),
-                '-c', os.path.join(TEST_DIR, '..', '..', config)])
+            # Create a configuration file to supply to the keystone
+            # server for the test run
+            with tempfile.NamedTemporaryFile() as conf_fp:
+                construct_temp_conf_file(conf_fp, config)
 
-            # blatent hack.
-            time.sleep(3)
-            if SERVER.poll() is not None:
-                print >> sys.stderr, 'Failed to start SERVER'
-                sys.exit(-1)
+                # Populate the test database
+                print "Populating registry and token databases..."
+                execute('sampledata.sh -c %s' % conf_fp.name)
 
-            try:
-                # run tests
-                subprocess.check_call(['unit2', 'discover', 'keystone.test'])
-            finally:
-                #kill the keystone SERVER
-                SERVER.kill()
+                # run the keystone server
+                print "Starting the keystone server..."
+                server = subprocess.Popen(
+                    [os.path.join(TEST_DIR, '../../bin/keystone'),
+                    '-c', conf_fp.name])
+
+                # blatent hack.
+                time.sleep(3)
+                if server.poll() is not None:
+                    print >> sys.stderr, 'Failed to start server'
+                    sys.exit(-1)
+
+                try:
+                    # discover and run tests
+                    print "Running tests..."
+                    execute('unit2 discover keystone.test')
+                finally:
+                    #kill the keystone server
+                    print "Stopping the keystone server..."
+                    server.kill()
         finally:
-            delete_temp_files()
+            remove_test_files()

keystone/xsd

@@ -1 +0,0 @@
-../doc/guide/src/docbkx/xsd

run_tests.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+function usage {
+  echo "Usage: $0 [OPTION]..."
+  echo "Run Keystone's test suite(s)"
+  echo ""
+  echo "  -V, --virtual-env        Always use virtualenv.  Install automatically if not present"
+  echo "  -N, --no-virtual-env     Don't use virtualenv.  Run tests in local environment"
+  echo "  -f, --force              Force a clean re-build of the virtual environment. Useful when dependencies have been added."
+  echo "  --unittests-only         Run unit tests only, exclude functional tests."
+  echo "  -p, --pep8               Just run pep8"
+  echo "  -h, --help               Print this usage message"
+  echo ""
+  echo "Note: with no options specified, the script will try to run the tests in a virtual environment,"
+  echo "      If no virtualenv is found, the script will ask if you would like to create one.  If you "
+  echo "      prefer to run tests NOT in a virtual environment, simply pass the -N option."
+  exit
+}
+
+function process_option {
+  case "$1" in
+    -h|--help) usage;;
+    -V|--virtual-env) let always_venv=1; let never_venv=0;;
+    -N|--no-virtual-env) let always_venv=0; let never_venv=1;;
+    -p|--pep8) let just_pep8=1;;
+    -f|--force) let force=1;;
+    --unittests-only) noseargs="$noseargs --exclude-dir=keystone/tests/functional --exclude-dir=keystone/tests/system";;
+    *) noseargs="$noseargs $1"
+  esac
+}
+
+venv=.keystone-venv
+with_venv=tools/with_venv.sh
+always_venv=0
+never_venv=0
+force=0
+noseargs=
+wrapper=""
+just_pep8=0
+
+for arg in "$@"; do
+  process_option $arg
+done
+
+function run_tests {
+  # Just run the test suites in current environment
+  ${wrapper} $NOSETESTS
+}
+
+function run_pep8 {
+  echo "Running pep8 ..."
+  # FIXME(sirp): bzr version-info is not currently pep-8. This was fixed with
+  # lp701898 [1], however, until that version of bzr becomes standard, I'm just
+  # excluding the vcsversion.py file
+  #
+  # [1] https://bugs.launchpad.net/bzr/+bug/701898
+  #
+  PEP8_EXCLUDE=vcsversion.py
+  PEP8_OPTIONS="--exclude=$PEP8_EXCLUDE --repeat --show-pep8 --show-source"
+  PEP8_INCLUDE="bin/* keystone tests tools setup.py run_tests.py"
+  pep8 $PEP8_OPTIONS $PEP8_INCLUDE
+}
+
+
+NOSETESTS="python keystone/test/run_tests.py $noseargs"
+
+if [ $never_venv -eq 0 ]
+then
+  # Remove the virtual environment if --force used
+  if [ $force -eq 1 ]; then
+    echo "Cleaning virtualenv..."
+    rm -rf ${venv}
+  fi
+  if [ -e ${venv} ]; then
+    wrapper="${with_venv}"
+  else
+    if [ $always_venv -eq 1 ]; then
+      # Automatically install the virtualenv
+      python tools/install_venv.py
+      wrapper="${with_venv}"
+    else
+      echo -e "No virtual environment found...create one? (Y/n) \c"
+      read use_ve
+      if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then
+        # Install the virtualenv and run the test suite in it
+        python tools/install_venv.py
+		    wrapper=${with_venv}
+      fi
+    fi
+  fi
+fi
+
+if [ $just_pep8 -eq 1 ]; then
+    run_pep8
+    exit
+fi
+
+run_tests || exit
+
+#if [ -z "$noseargs" ]; then
+#  run_pep8
+#fi

setup.py

@@ -54,6 +54,7 @@ setup(
     zip_safe=False,
     cmdclass=cmdclass,
     install_requires=['setuptools'],
+    test_suite='nose.collector',
     entry_points={
         'paste.app_factory': ['main=identity:app_factory'],
         'paste.filter_factory': [

tools/install_venv.py

@@ -0,0 +1,140 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+# Copyright 2010 OpenStack LLC.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Installation script for Keystone's development virtualenv
+"""
+
+import os
+import subprocess
+import sys
+
+
+ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+VENV = os.path.join(ROOT, '.keystone-venv')
+PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires')
+
+
+def die(message, *args):
+    print >> sys.stderr, message % args
+    sys.exit(1)
+
+
+def run_command(cmd, redirect_output=True, check_exit_code=True):
+    """
+    Runs a command in an out-of-process shell, returning the
+    output of that command.  Working directory is ROOT.
+    """
+    if redirect_output:
+        stdout = subprocess.PIPE
+    else:
+        stdout = None
+
+    proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout)
+    output = proc.communicate()[0]
+    if check_exit_code and proc.returncode != 0:
+        die('Command "%s" failed.\n%s', ' '.join(cmd), output)
+    return output
+
+
+HAS_EASY_INSTALL = bool(run_command(['which', 'easy_install'],
+                                    check_exit_code=False).strip())
+HAS_VIRTUALENV = bool(run_command(['which', 'virtualenv'],
+                                    check_exit_code=False).strip())
+
+
+def check_dependencies():
+    """Make sure virtualenv is in the path."""
+
+    if not HAS_VIRTUALENV:
+        print 'not found.'
+        # Try installing it via easy_install...
+        if HAS_EASY_INSTALL:
+            print 'Installing virtualenv via easy_install...',
+            if not run_command(['which', 'easy_install']):
+                die('ERROR: virtualenv not found.\n\n'
+                    'Keystone development requires virtualenv, please install'
+                    ' it using your favorite package management tool')
+            print 'done.'
+    print 'done.'
+
+
+def create_virtualenv(venv=VENV):
+    """
+    Creates the virtual environment and installs PIP only into the
+    virtual environment
+    """
+    print 'Creating venv...',
+    run_command(['virtualenv', '-q', '--no-site-packages', VENV])
+    print 'done.'
+    print 'Installing pip in virtualenv...',
+    if not run_command(['tools/with_venv.sh', 'easy_install', 'pip']).strip():
+        die("Failed to install pip.")
+    print 'done.'
+
+
+def install_dependencies(venv=VENV):
+    print 'Installing dependencies with pip (this can take a while)...'
+
+    # Install greenlet by hand - just listing it in the requires file does not
+    # get it in stalled in the right order
+    venv_tool = 'tools/with_venv.sh'
+    run_command([venv_tool, 'pip', 'install', '-E', venv, '-r', PIP_REQUIRES],
+                redirect_output=False)
+
+    # Tell the virtual env how to "import keystone"
+
+    for version in ['python2.7', 'python2.6']:
+        pth = os.path.join(venv, "lib", version, "site-packages")
+        if os.path.exists(pth):
+            pthfile = os.path.join(pth, "keystone.pth")
+            f = open(pthfile, 'w')
+            f.write("%s\n" % ROOT)
+
+def print_help():
+    help = """
+ Keystone development environment setup is complete.
+
+ Keystone development uses virtualenv to track and manage Python dependencies
+ while in development and testing.
+
+ To activate the Keystone virtualenv for the extent of your current shell
+ session you can run:
+
+ $ source .keystone-venv/bin/activate
+
+ Or, if you prefer, you can run commands in the virtualenv on a case by case
+ basis by running:
+
+ $ tools/with_venv.sh <your command>
+
+ Also, make test will automatically use the virtualenv.
+    """
+    print help
+
+
+def main(argv):
+    check_dependencies()
+    create_virtualenv()
+    install_dependencies()
+    print_help()
+
+if __name__ == '__main__':
+    main(sys.argv)

tools/pip-requires

@@ -1,3 +1,8 @@
+# You may need to install development files before using 'pip install'
+# For example:
+#   sudo apt-get install python-dev libxml2-dev libxslt1-dev libsasl2d-ev libldap2-dev libsqlite3-dev
+
+# Production
 eventlet
 lxml
 paste
@@ -8,5 +13,18 @@ sqlalchemy
 webob
 Routes
 httplib2
-python-ldap==2.3.13 # optional authentication backend (may require OpenLDAP libs)
+
+# Optional backend: LDAP
+python-ldap
+
+# Optional backend: Memcache
 python-memcached
+
+# Development
+Sphinx # required to build documentation
+coverage # computes code coverage percentages
+
+# Testing
+webtest
+unittest2
+pep8

tools/pip-requires-development

@@ -1,2 +0,0 @@
-Sphinx # required to build documentation
-coverage # computes code coverage percentages

tools/pip-requires-testing

@@ -1,3 +0,0 @@
-webtest
-DTest
-unittest2

tools/with_venv.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+TOOLS=`dirname $0`
+VENV=$TOOLS/../.keystone-venv
+source $VENV/bin/activate && $@