Begin adding testing functionality

Take the current run_tests.sh from glance and include
that as well as fill in the rest of the pieces needed
for those that use venv functionality. Start adding
memory tests.

openstack-common.conf

@@ -1,7 +1,7 @@
 [DEFAULT]
 
 # The list of modules to copy from oslo-incubator.git
-modules=excutils
+modules=excutils,install_venv_common
 
 # The base module to hold the copy of openstack.common
 base=taskflow

run_tests.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+
+function usage {
+  echo "Usage: $0 [OPTION]..."
+  echo "Run Taskflow's test suite(s)"
+  echo ""
+  echo "  -V, --virtual-env        Always use virtualenv.  Install automatically if not present"
+  echo "  -N, --no-virtual-env     Don't use virtualenv.  Run tests in local environment"
+  echo "  -f, --force              Force a clean re-build of the virtual environment. Useful when dependencies have been added."
+  echo "  -u, --update             Update the virtual environment with any newer package versions"
+  echo "  -p, --pep8               Just run pep8"
+  echo "  -P, --no-pep8            Don't run static code checks"
+  echo "  -h, --help               Print this usage message"
+  echo ""
+  echo "Note: with no options specified, the script will try to run the tests in a virtual environment,"
+  echo "      If no virtualenv is found, the script will ask if you would like to create one.  If you "
+  echo "      prefer to run tests NOT in a virtual environment, simply pass the -N option."
+  exit
+}
+
+function process_option {
+  case "$1" in
+    -h|--help) usage;;
+    -V|--virtual-env) let always_venv=1; let never_venv=0;;
+    -N|--no-virtual-env) let always_venv=0; let never_venv=1;;
+    -p|--pep8) let just_pep8=1;;
+    -P|--no-pep8) let no_pep8=1;;
+    -f|--force) let force=1;;
+    -u|--update) update=1;;
+    -c|--coverage) noseopts="$noseopts --with-coverage --cover-package=taskflow";;
+    -*) noseopts="$noseopts $1";;
+    *) noseargs="$noseargs $1"
+  esac
+}
+
+venv=.venv
+with_venv=tools/with_venv.sh
+always_venv=0
+never_venv=0
+force=0
+noseopts=
+noseargs=
+wrapper=""
+just_pep8=0
+no_pep8=0
+update=0
+
+export NOSE_WITH_OPENSTACK=1
+export NOSE_OPENSTACK_COLOR=1
+export NOSE_OPENSTACK_RED=0.05
+export NOSE_OPENSTACK_YELLOW=0.025
+export NOSE_OPENSTACK_SHOW_ELAPSED=1
+export NOSE_OPENSTACK_STDOUT=1
+
+for arg in "$@"; do
+  process_option $arg
+done
+
+function run_tests {
+  # Cleanup *pyc
+  ${wrapper} find . -type f -name "*.pyc" -delete
+  ${wrapper} $NOSETESTS
+}
+
+function run_pep8 {
+  echo "Running pep8 ..."
+  PEP8_EXCLUDE=".venv,.tox,dist,doc,openstack"
+  PEP8_OPTIONS="--exclude=$PEP8_EXCLUDE --repeat"
+  PEP8_IGNORE="--ignore=E125,E126,E711,E712"
+  PEP8_INCLUDE="."
+
+  ${wrapper} pep8 $PEP8_OPTIONS $PEP8_INCLUDE $PEP8_IGNORE
+}
+
+
+NOSETESTS="nosetests $noseopts $noseargs"
+
+if [ $never_venv -eq 0 ]
+then
+  # Remove the virtual environment if --force used
+  if [ $force -eq 1 ]; then
+    echo "Cleaning virtualenv..."
+    rm -rf ${venv}
+  fi
+  if [ $update -eq 1 ]; then
+    echo "Updating virtualenv..."
+    python tools/install_venv.py
+  fi
+  if [ -e ${venv} ]; then
+    wrapper="${with_venv}"
+  else
+    if [ $always_venv -eq 1 ]; then
+      # Automatically install the virtualenv
+      python tools/install_venv.py
+      wrapper="${with_venv}"
+    else
+      echo -e "No virtual environment found...create one? (Y/n) \c"
+      read use_ve
+      if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then
+        # Install the virtualenv and run the test suite in it
+        python tools/install_venv.py
+		wrapper=${with_venv}
+      fi
+    fi
+  fi
+fi
+
+if [ $just_pep8 -eq 1 ]; then
+    run_pep8
+    exit
+fi
+
+run_tests || exit
+
+if [ -z "$noseargs" ]; then
+    if [ $no_pep8 -eq 0 ]; then
+        run_pep8
+    fi
+fi

setup.py

@@ -18,7 +18,8 @@ def read_requires(base):
     return requires
 
 
-setuptools.setup(name='taskflow',
+setuptools.setup(
+    name='taskflow',
     version='0.0.1',
     author='OpenStack',
     license='Apache Software License',
@@ -36,4 +37,4 @@ setuptools.setup(name='taskflow',
         'License :: OSI Approved :: Apache Software License',
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python :: 2.6', ],
-)
+)

taskflow/backends/memory.py

@@ -24,6 +24,7 @@ import threading
 
 from taskflow import catalog
 from taskflow import exceptions as exc
+from taskflow import job
 from taskflow import jobboard
 from taskflow import logbook
 from taskflow import utils
@@ -43,6 +44,14 @@ def check_not_closed(meth):
     return check
 
 
+class MemoryClaimer(job.Claimer):
+    def claim(self, job, owner):
+        # Straight foward check if already claimed.
+        if job.owner is not None:
+            raise exc.UnclaimableJobException()
+        job.owner = owner
+
+
 class MemoryCatalog(catalog.Catalog):
     def __init__(self):
         super(MemoryCatalog, self).__init__()
@@ -50,6 +59,13 @@ class MemoryCatalog(catalog.Catalog):
         self._closed = False
         self._lock = threading.RLock()
 
+    def __contains__(self, job):
+        with self._lock:
+            for (j, b) in self._catalogs:
+                if j == job:
+                    return True
+        return False
+
     def close(self):
         self._closed = True
 
@@ -122,12 +138,11 @@ class MemoryJobBoard(jobboard.JobBoard):
     @check_not_closed
     def post(self, job):
         with self._lock.acquire(read=False):
-            self._active_jobs.append((datetime.utcnow(), job))
+            self._board.append((datetime.utcnow(), job))
         # Let people know a job is here
         self._notify_posted(job)
         self._event.set()
-        # And now that they are notified, wait for
-        # another posting.
+        # And now that they are notified, wait for another posting.
         self._event.clear()
 
     @check_not_closed
@@ -148,9 +163,9 @@ class MemoryJobBoard(jobboard.JobBoard):
                 if j == job:
                     exists = True
                     break
-           if not exists:
-               raise exc.JobNotFound()
-           self._board = [(d, j) for (d, j) in self._board if j != job]
+            if not exists:
+                raise exc.JobNotFound()
+            self._board = [(d, j) for (d, j) in self._board if j != job]
 
     @check_not_closed
     def posted_after(self, date_posted=None):
@@ -163,5 +178,4 @@ class MemoryJobBoard(jobboard.JobBoard):
 
     @check_not_closed
     def await(self, timeout=None):
-        with not self._event.is_set():
-            self._event.wait(timeout)
+        self._event.wait(timeout)

taskflow/job.py

@@ -39,11 +39,9 @@ class Claimer(object):
 
 class Job(object):
     """A job is connection to some set of work to be done by some agent. Basic
-    information is provided about said work to be able to attempt to 
+    information is provided about said work to be able to attempt to
     fullfill said work."""
 
-    __metaclass__ = abc.ABCMeta
-
     def __init__(self, name, context, catalog, claimer):
         self.name = name
         self.context = context
@@ -79,7 +77,7 @@ class Job(object):
         self._claimer.claim(self, owner)
         self.owner = owner
         self._change_state(states.CLAIMED)
- 
+
     def _change_state(self, new_state):
         self.state = new_state
         # TODO(harlowja): update the logbook

taskflow/tests/__init__.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

taskflow/tests/unit/__init__.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

taskflow/tests/unit/test_memory.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from datetime import datetime
+
+import time
+import threading
+import unittest
+
+from taskflow import job
+from taskflow.backends import memory
+
+
+class MemoryBackendTest(unittest.TestCase):
+    def testPostRecvJob(self):
+        job_claimer = memory.MemoryClaimer()
+        book_catalog = memory.MemoryCatalog()
+        j = job.Job("test", {}, book_catalog, job_claimer)
+
+        # Hook up some simulated workers to said job-board.
+        job_board = memory.MemoryJobBoard()
+        receiver_awake = threading.Event()
+        work_items = []
+
+        def post_job():
+            job_board.post(j)
+
+        def work_on_job(j):
+            owner = 'me'
+            j.claim(owner)
+
+        def receive_job():
+            start = datetime.utcnow()
+            receiver_awake.set()
+            new_jobs = []
+            while not new_jobs:
+                job_board.await(0.5)
+                new_jobs = job_board.posted_after(start)
+            work_items.extend(new_jobs)
+            for j in work_items:
+                work_on_job(j)
+
+        poster = threading.Thread(target=post_job)
+        receiver = threading.Thread(target=receive_job)
+        receiver.start()
+        while not receiver_awake.isSet():
+            receiver_awake.wait()
+        poster.start()
+
+        for t in [poster, receiver]:
+            t.join()
+
+        self.assertEquals(1, len(work_items))
+        self.assertEquals(j.owner, 'me')

tools/install_venv.py

@@ -0,0 +1,76 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+# Copyright 2010 OpenStack LLC.
+# Copyright 2013 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Installation script for Taskflows's development virtualenv
+"""
+
+import os
+import subprocess
+import sys
+
+import install_venv_common as install_venv
+
+
+def print_help():
+    help = """
+Taskflow development environment setup is complete.
+
+Taskflow development uses virtualenv to track and manage Python dependencies
+while in development and testing.
+
+To activate the taskflow virtualenv for the extent of your current shell
+session you can run:
+
+$ source .venv/bin/activate
+
+Or, if you prefer, you can run commands in the virtualenv on a case by case
+basis by running:
+
+$ tools/with_venv.sh <your command>
+
+Also, make test will automatically use the virtualenv.
+    """
+    for line in help.splitlines():
+        print(" %s" % (line))
+
+
+def main(argv):
+    root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+    venv = os.path.join(root, '.venv')
+    pip_requires = os.path.join(root, 'tools', 'pip-requires')
+    test_requires = os.path.join(root, 'tools', 'test-requires')
+    py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
+    project = 'Taskflow'
+    install = install_venv.InstallVenv(root, venv, pip_requires, test_requires,
+                                       py_version, project)
+    options = install.parse_args(argv)
+    install.check_python_version()
+    install.check_dependencies()
+    install.create_virtualenv(no_site_packages=options.no_site_packages)
+    install.install_dependencies()
+    install.run_command([os.path.join(venv, 'bin/python'),
+                        'setup.py', 'develop'])
+    install.post_process()
+    print_help()
+
+if __name__ == '__main__':
+    main(sys.argv)

tools/install_venv_common.py

@@ -0,0 +1,221 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 OpenStack Foundation
+# Copyright 2013 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Provides methods needed by installation script for OpenStack development
+virtual environments.
+
+Since this script is used to bootstrap a virtualenv from the system's Python
+environment, it should be kept strictly compatible with Python 2.6.
+
+Synced in from openstack-common
+"""
+
+from __future__ import print_function
+
+import optparse
+import os
+import subprocess
+import sys
+
+
+class InstallVenv(object):
+
+    def __init__(self, root, venv, pip_requires, test_requires, py_version,
+                 project):
+        self.root = root
+        self.venv = venv
+        self.pip_requires = pip_requires
+        self.test_requires = test_requires
+        self.py_version = py_version
+        self.project = project
+
+    def die(self, message, *args):
+        print(message % args, file=sys.stderr)
+        sys.exit(1)
+
+    def check_python_version(self):
+        if sys.version_info < (2, 6):
+            self.die("Need Python Version >= 2.6")
+
+    def run_command_with_code(self, cmd, redirect_output=True,
+                              check_exit_code=True):
+        """Runs a command in an out-of-process shell.
+
+        Returns the output of that command. Working directory is self.root.
+        """
+        if redirect_output:
+            stdout = subprocess.PIPE
+        else:
+            stdout = None
+
+        proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout)
+        output = proc.communicate()[0]
+        if check_exit_code and proc.returncode != 0:
+            self.die('Command "%s" failed.\n%s', ' '.join(cmd), output)
+        return (output, proc.returncode)
+
+    def run_command(self, cmd, redirect_output=True, check_exit_code=True):
+        return self.run_command_with_code(cmd, redirect_output,
+                                          check_exit_code)[0]
+
+    def get_distro(self):
+        if (os.path.exists('/etc/fedora-release') or
+                os.path.exists('/etc/redhat-release')):
+            return Fedora(self.root, self.venv, self.pip_requires,
+                          self.test_requires, self.py_version, self.project)
+        else:
+            return Distro(self.root, self.venv, self.pip_requires,
+                          self.test_requires, self.py_version, self.project)
+
+    def check_dependencies(self):
+        self.get_distro().install_virtualenv()
+
+    def create_virtualenv(self, no_site_packages=True):
+        """Creates the virtual environment and installs PIP.
+
+        Creates the virtual environment and installs PIP only into the
+        virtual environment.
+        """
+        if not os.path.isdir(self.venv):
+            print('Creating venv...', end=' ')
+            if no_site_packages:
+                self.run_command(['virtualenv', '-q', '--no-site-packages',
+                                 self.venv])
+            else:
+                self.run_command(['virtualenv', '-q', self.venv])
+            print('done.')
+            print('Installing pip in venv...', end=' ')
+            if not self.run_command(['tools/with_venv.sh', 'easy_install',
+                                    'pip>1.0']).strip():
+                self.die("Failed to install pip.")
+            print('done.')
+        else:
+            print("venv already exists...")
+            pass
+
+    def pip_install(self, *args):
+        self.run_command(['tools/with_venv.sh',
+                         'pip', 'install', '--upgrade'] + list(args),
+                         redirect_output=False)
+
+    def install_dependencies(self):
+        print('Installing dependencies with pip (this can take a while)...')
+
+        # First things first, make sure our venv has the latest pip and
+        # distribute.
+        # NOTE: we keep pip at version 1.1 since the most recent version causes
+        # the .venv creation to fail. See:
+        # https://bugs.launchpad.net/nova/+bug/1047120
+        self.pip_install('pip==1.1')
+        self.pip_install('distribute')
+
+        # Install greenlet by hand - just listing it in the requires file does
+        # not
+        # get it installed in the right order
+        self.pip_install('greenlet')
+
+        self.pip_install('-r', self.pip_requires)
+        self.pip_install('-r', self.test_requires)
+
+    def post_process(self):
+        self.get_distro().post_process()
+
+    def parse_args(self, argv):
+        """Parses command-line arguments."""
+        parser = optparse.OptionParser()
+        parser.add_option('-n', '--no-site-packages',
+                          action='store_true',
+                          help="Do not inherit packages from global Python "
+                               "install")
+        return parser.parse_args(argv[1:])[0]
+
+
+class Distro(InstallVenv):
+
+    def check_cmd(self, cmd):
+        return bool(self.run_command(['which', cmd],
+                    check_exit_code=False).strip())
+
+    def install_virtualenv(self):
+        if self.check_cmd('virtualenv'):
+            return
+
+        if self.check_cmd('easy_install'):
+            print('Installing virtualenv via easy_install...', end=' ')
+            if self.run_command(['easy_install', 'virtualenv']):
+                print('Succeeded')
+                return
+            else:
+                print('Failed')
+
+        self.die('ERROR: virtualenv not found.\n\n%s development'
+                 ' requires virtualenv, please install it using your'
+                 ' favorite package management tool' % self.project)
+
+    def post_process(self):
+        """Any distribution-specific post-processing gets done here.
+
+        In particular, this is useful for applying patches to code inside
+        the venv.
+        """
+        pass
+
+
+class Fedora(Distro):
+    """This covers all Fedora-based distributions.
+
+    Includes: Fedora, RHEL, CentOS, Scientific Linux
+    """
+
+    def check_pkg(self, pkg):
+        return self.run_command_with_code(['rpm', '-q', pkg],
+                                          check_exit_code=False)[1] == 0
+
+    def apply_patch(self, originalfile, patchfile):
+        self.run_command(['patch', '-N', originalfile, patchfile],
+                         check_exit_code=False)
+
+    def install_virtualenv(self):
+        if self.check_cmd('virtualenv'):
+            return
+
+        if not self.check_pkg('python-virtualenv'):
+            self.die("Please install 'python-virtualenv'.")
+
+        super(Fedora, self).install_virtualenv()
+
+    def post_process(self):
+        """Workaround for a bug in eventlet.
+
+        This currently affects RHEL6.1, but the fix can safely be
+        applied to all RHEL and Fedora distributions.
+
+        This can be removed when the fix is applied upstream.
+
+        Nova: https://bugs.launchpad.net/nova/+bug/884915
+        Upstream: https://bitbucket.org/which_linden/eventlet/issue/89
+        """
+
+        # Install "patch" program if it's not there
+        if not self.check_pkg('patch'):
+            self.die("Please install 'patch'.")
+
+        # Apply the eventlet patch
+        self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
+                                      'site-packages',
+                                      'eventlet/green/subprocess.py'),
+                         'contrib/redhat-eventlet.patch')

tools/pip-requires

@@ -1 +1,3 @@
-oslo.cfg>=1.1.0
+# Packages needed for using this library.
+
+oslo.config>=1.1.0

tools/test-requires

@@ -0,0 +1,8 @@
+# Packages needed for testing this library.
+
+nose
+pep8==1.4.5
+nose-exclude
+openstack.nose_plugin>=0.7
+mox
+coverage

tools/with_venv.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+TOOLS=`dirname $0`
+VENV=$TOOLS/../.venv
+source $VENV/bin/activate && $@