Add ability to debug OVS virtual environments

When writing tests, it is useful to be able to look at the database
being modified. Since we use OVS virtual environments and delete
them when the test ends, this is difficult to do.

This patch adds the ability to pass KEEP_ENV=1 when running tox to
keep the virtual environment around and adds tools/debug_venv to
launch an OVS sandbox using the OVS virtual environment.

Change-Id: Idbec2d544d913ffa8888a050b8a53d016c0315f2

TESTING.rst

@@ -0,0 +1,58 @@
+ovsdbapp testing
+================
+
+Test preferences
+----------------
+Most ovsdbapp tests will be functional tests. Unit tests are reserved primarily
+for functions that produce a given output for their inputs, regardless of
+externalities. Unit tests using mock are acceptable if something is hard to
+test without it. BUT, please think carefully before writing a test that makes
+heavy use of mock.assert_called_once_with() as those tests *very* often tend
+to test what a function *currently does* and not what a function *should do*.
+
+
+Running tests
+-------------
+Tests are run with tox. Generally in the form of:
+
+.. code-block:: shell
+
+    tox -e <test-type>
+
+
+Functional tests
+----------------
+Run the functional tests with:
+
+.. code-block:: shell
+
+    tox -e functional
+
+The ovsdbapp functional tests create an OVS virtual environment similar to a
+Python virtualenv. OVS will be checked out from git and placed in
+.tox/functional/src/ovs and a virtual environment directory will be created.
+Various OVS servers will be launched and will store their runtime files in
+the virtual environment directory. The tests will then be run against these
+servers. Upon test completion, the servers will be killed and the virtual
+environment directory deleted. Note that one environment is created per test
+process, by default one per-core.
+
+In the event that debugging tests is necessary, it is possible to keep the
+virtual environment directories by running:
+
+.. code-block:: shell
+
+    KEEP_VENV=1 tox -e functional
+
+This will also write an informational file .tox/functional/ovsvenv.$pid for
+each process. The first line of this file is the virtual environment directory
+and additional lines are the tests run by this process. To load an OVS
+virtualenv for debugging for a particular test (e.g. test_ls_add_name), call:
+
+.. code-block:: shell
+
+    tools/debug_venv test_ls_add_name
+
+This will spawn a shell where you can run ovs-vsctl, ovn-nbctl, etc. on the db
+used by the test. When finished, type 'exit'. See the debug_venv help for more
+options.

ovsdbapp/tests/functional/base.py

@@ -23,9 +23,21 @@ from ovsdbapp import venv
 class FunctionalTestCase(base.TestCase):
     connection = None
     ovsvenv = venv.OvsVenvFixture(tempfile.mkdtemp(),
-                                  ovsdir=os.getenv('OVS_SRCDIR'), remove=True)
+                                  ovsdir=os.getenv('OVS_SRCDIR'),
+                                  remove=not bool(os.getenv('KEEP_VENV')))
     atexit.register(ovsvenv.cleanUp)
     ovsvenv.setUp()
+    ovsvenvlog = None
+    if os.getenv('KEEP_VENV') and os.getenv('VIRTUAL_ENV'):
+        ovsvenvlog = open(os.path.join(os.getenv('VIRTUAL_ENV'),
+                                       'ovsvenv.%s' % os.getpid()), 'a+')
+        atexit.register(ovsvenvlog.close)
+        ovsvenvlog.write("%s\n" % ovsvenv.venv)
+
+    @classmethod
+    def venv_log(cls, val):
+        if cls.ovsvenvlog:
+            cls.ovsvenvlog.write("%s\n" % val)
 
     @classmethod
     def set_connection(cls):
@@ -44,4 +56,5 @@ class FunctionalTestCase(base.TestCase):
 
     def setUp(self):
         super(FunctionalTestCase, self).setUp()
+        self.venv_log(self.id())
         self.set_connection()

tools/debug_venv

@@ -0,0 +1,63 @@
+#!/bin/sh
+# Use a shell script to launch tools/debub_venv.py so that we can enter a
+# virtual environment if we are not already in one
+
+for option;do
+    if test -n "$prev";then
+        eval $prev=\$option
+        prev=
+        continue
+    fi
+    case $option in
+        -v)
+            if [ -n "$VIRTUAL_ENV" ]; then
+                echo "Already in a virtual environment" 1>&2
+                exit 1
+            fi
+            prev=venv ;;
+        -o)
+            prev=ovsvenv ;;
+        -h|--help)
+            cat << EOF
+debug_venv: debug a test OVS virtual environment
+usage: debug_venv [-v virtualenv] [-o ovsvenv | test_regex]"
+
+Options:
+  -v  The Python virtualenv to enter (defaults to 'functional')
+  -o  The OVS virtual environment directory (precludes test_regex)
+  test_regex An optionsal regular expression matching the test name to debug
+EOF
+            exit ;;
+        *)
+            if test -z "$regex";then
+                regex=$option
+            else
+                echo "Only one regex" 1>&2
+                exit 1
+            fi
+    esac
+done
+
+if [ -z "$regex" -a -z "$ovsvenv" ]; then
+    echo "Need regex or ovsvenv" 1>&2
+    exit 1
+fi
+
+if [ -z "$VIRTUAL_ENV" ]; then
+    . .tox/${venv:-functional}/bin/activate
+    trap deactivate EXIT
+fi
+
+if [ -n "$regex" -a -z "$ovsvenv" ]; then
+    # Just do the first match for now
+    lookup=$(grep $regex $VIRTUAL_ENV/ovsvenv.*|head -1)
+    if [ -z "$lookup" ]; then
+        echo "Could not match $regex" 1>&2
+        exit 1
+    fi
+    test_file=$(echo $lookup|cut -d: -f1)
+    test_match=", matched $(echo $lookup|rev|cut -d: -f1|rev)"
+    ovsvenv=$(head -1 $test_file)
+fi
+echo "Debugging OVS virtual environment: $ovsvenv$test_match"
+tools/debug_venv.py $ovsvenv $VIRTUAL_ENV/src/ovs

tools/debug_venv.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from __future__ import print_function
+import atexit
+import os
+import subprocess
+import sys
+
+from fixtures import fixture
+
+from ovsdbapp import venv
+
+if len(sys.argv) != 3:
+    print("Requires two arguments: venvdir ovsdir", file=sys.stderr)
+    sys.exit(1)
+
+for d in sys.argv[1:]:
+    if not os.path.isdir(d):
+        print("%s is not a directory" % d, file=sys.stderr)
+        sys.exit(1)
+
+venvdir = os.path.abspath(sys.argv[1])
+ovsdir = os.path.abspath(sys.argv[2])
+
+v = venv.OvsVenvFixture(venvdir, ovsdir)
+try:
+    atexit.register(v.cleanUp)
+    v.setUp()
+except fixture.MultipleExceptions as e:
+    raise e.args[0][0], e.args[0][1], e.args[0][2]
+try:
+    print("*** Exit the shell when finished debugging ***")
+    subprocess.call([os.getenv('SHELL'), '-i'], env=v.env)
+except Exception:
+    print("*** Could not start shell, don't type 'exit'***", file=sys.stderr)
+    raise

tox.ini

@@ -38,6 +38,7 @@ commands = oslo_debug_helper {posargs}
 setenv = {[testenv]setenv}
          OS_TEST_PATH=./ovsdbapp/tests/functional
          OVS_SRCDIR={envdir}/src/ovs
+passenv = KEEP_VENV
 
 [flake8]
 # E123, E125 skipped as they are invalid PEP-8.