Add post-mortem debug option for tests

Post-mortem debugging, the ability to drop into a debugger with the
execution state that triggered the exception, is very useful in
diagnosing failure conditions.  Our previous test runner, nose,
provided the ability to enable post-mortem debugging on test
failures (via --pdb-failure) and errors (via --pdb).  testr
lacks these options at present, so this change adds support
for enabling post-mortem debugging via an environment variable.
All test-triggered exceptions will result in a post-mortem debugger
being invoked if OS_POST_MORTEM_DEBUG is set to "1" or "True".

Implements: blueprint neutron-pm-debug-on-test-failure
Change-Id: Iddbe1335b059d062c0286df2ad27aef7728461b7

TESTING

@@ -62,3 +62,39 @@ Development process
     fixed!  In addition, before proposing for merge, all of the
     current tests should be passing.
 
+Debugging
+
+    By default, calls to pdb.set_trace() will be ignored when tests
+    are run.  For pdb statements to work, invoke run_tests as follows:
+
+    $ ./run_tests.sh -d [test module path]
+
+    It's possible to debug tests in a tox environment:
+
+    $ tox -e venv -- python -m testtools.run [test module path]
+
+    Tox-created virtual environments (venv's) can also be activated
+    after a tox run and reused for debugging:
+
+    $ tox -e venv
+    $ . .tox/venv/bin/activate
+    $ python -m testtools.run [test module path]
+
+    Tox packages and installs the neutron source tree in a given venv
+    on every invocation, but if modifications need to be made between
+    invocation (e.g. adding more pdb statements), it is recommended
+    that the source tree be installed in the venv in editable mode:
+
+    # run this only after activating the venv
+    $ pip install --editable .
+
+    Editable mode ensures that changes made to the source tree are
+    automatically reflected in the venv, and that such changes are not
+    overwritten during the next tox run.
+
+Post-mortem debugging
+
+    Setting OS_POST_MORTEM_DEBUG=1 in the shell environment will ensure
+    that pdb.post_mortem() will be invoked on test failure:
+
+    $ OS_POST_MORTEM_DEBUG=1 ./run_tests.sh -d [test module path]

neutron/tests/base.py

@@ -26,6 +26,8 @@ import fixtures
 from oslo.config import cfg
 import testtools
 
+from neutron.tests import post_mortem_debug
+
 
 CONF = cfg.CONF
 TRUE_STRING = ['True', '1']
@@ -41,6 +43,10 @@ class BaseTestCase(testtools.TestCase):
     def setUp(self):
         super(BaseTestCase, self).setUp()
 
+        # Configure this first to ensure pm debugging support for setUp()
+        if os.environ.get('OS_POST_MORTEM_DEBUG') in TRUE_STRING:
+            self.addOnException(post_mortem_debug.exception_handler)
+
         if os.environ.get('OS_DEBUG') in TRUE_STRING:
             _level = logging.DEBUG
         else:

neutron/tests/post_mortem_debug.py

@@ -0,0 +1,106 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import pdb
+import traceback
+
+
+def exception_handler(exc_info):
+    """Exception handler enabling post-mortem debugging.
+
+    A class extending testtools.TestCase can add this handler in setUp():
+
+        self.addOnException(post_mortem_debug.exception_handler)
+
+    When an exception occurs, the user will be dropped into a pdb
+    session in the execution environment of the failure.
+
+    Frames associated with the testing framework are excluded so that
+    the post-mortem session for an assertion failure will start at the
+    assertion call (e.g. self.assertTrue) rather than the framework code
+    that raises the failure exception (e.g. the assertTrue method).
+    """
+    tb = exc_info[2]
+    ignored_traceback = get_ignored_traceback(tb)
+    if ignored_traceback:
+        tb = FilteredTraceback(tb, ignored_traceback)
+    traceback.print_exception(exc_info[0], exc_info[1], tb)
+    pdb.post_mortem(tb)
+
+
+def get_ignored_traceback(tb):
+    """Retrieve the first traceback of an ignored trailing chain.
+
+    Given an initial traceback, find the first traceback of a trailing
+    chain of tracebacks that should be ignored.  The criteria for
+    whether a traceback should be ignored is whether its frame's
+    globals include the __unittest marker variable. This criteria is
+    culled from:
+
+        unittest.TestResult._is_relevant_tb_level
+
+    For example:
+
+       tb.tb_next => tb0.tb_next => tb1.tb_next
+
+    - If no tracebacks were to be ignored, None would be returned.
+    - If only tb1 was to be ignored, tb1 would be returned.
+    - If tb0 and tb1 were to be ignored, tb0 would be returned.
+    - If either of only tb or only tb0 was to be ignored, None would
+      be returned because neither tb or tb0 would be part of a
+      trailing chain of ignored tracebacks.
+    """
+    # Turn the traceback chain into a list
+    tb_list = []
+    while tb:
+        tb_list.append(tb)
+        tb = tb.tb_next
+
+    # Find all members of an ignored trailing chain
+    ignored_tracebacks = []
+    for tb in reversed(tb_list):
+        if '__unittest' in tb.tb_frame.f_globals:
+            ignored_tracebacks.append(tb)
+        else:
+            break
+
+    # Return the first member of the ignored trailing chain
+    if ignored_tracebacks:
+        return ignored_tracebacks[-1]
+
+
+class FilteredTraceback(object):
+    """Wraps a traceback to filter unwanted frames."""
+
+    def __init__(self, tb, filtered_traceback):
+        """Constructor.
+
+        :param tb: The start of the traceback chain to filter.
+        :param filtered_traceback: The first traceback of a trailing
+               chain that is to be filtered.
+        """
+        self._tb = tb
+        self.tb_lasti = self._tb.tb_lasti
+        self.tb_lineno = self._tb.tb_lineno
+        self.tb_frame = self._tb.tb_frame
+        self._filtered_traceback = filtered_traceback
+
+    @property
+    def tb_next(self):
+        tb_next = self._tb.tb_next
+        if tb_next and tb_next != self._filtered_traceback:
+            return FilteredTraceback(tb_next, self._filtered_traceback)

neutron/tests/unit/test_post_mortem_debug.py

@@ -0,0 +1,98 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sys
+
+import mock
+
+from neutron.tests import base
+from neutron.tests import post_mortem_debug
+
+
+class TestTesttoolsExceptionHandler(base.BaseTestCase):
+
+    def test_exception_handler(self):
+        try:
+            self.assertTrue(False)
+        except Exception:
+            exc_info = sys.exc_info()
+        with mock.patch('traceback.print_exception') as mock_print_exception:
+            with mock.patch('pdb.post_mortem') as mock_post_mortem:
+                with mock.patch.object(post_mortem_debug,
+                                       'get_ignored_traceback',
+                                       return_value=mock.Mock()):
+                    post_mortem_debug.exception_handler(exc_info)
+
+        mock_print_exception.called_once_with(*exc_info)
+        mock_post_mortem.called_once()
+
+
+class TestFilteredTraceback(base.BaseTestCase):
+
+    def test_filter_traceback(self):
+        tb1 = mock.Mock()
+        tb2 = mock.Mock()
+        tb1.tb_next = tb2
+        tb2.tb_next = None
+        ftb1 = post_mortem_debug.FilteredTraceback(tb1, tb2)
+        for attr in ['lasti', 'lineno', 'frame']:
+            attr_name = 'tb_%s' % attr
+            self.assertEqual(getattr(tb1, attr_name, None),
+                             getattr(ftb1, attr_name, None))
+        self.assertIsNone(ftb1.tb_next)
+
+
+class TestGetIgnoredTraceback(base.BaseTestCase):
+
+    def _test_get_ignored_traceback(self, ignored_bit_array, expected):
+        root_tb = mock.Mock()
+
+        tb = root_tb
+        tracebacks = [tb]
+        for x in xrange(len(ignored_bit_array) - 1):
+            tb.tb_next = mock.Mock()
+            tb = tb.tb_next
+            tracebacks.append(tb)
+        tb.tb_next = None
+
+        tb = root_tb
+        for ignored in ignored_bit_array:
+            if ignored:
+                tb.tb_frame.f_globals = ['__unittest']
+            else:
+                tb.tb_frame.f_globals = []
+            tb = tb.tb_next
+
+        actual = post_mortem_debug.get_ignored_traceback(root_tb)
+        if expected is not None:
+            expected = tracebacks[expected]
+        self.assertEqual(actual, expected)
+
+    def test_no_ignored_tracebacks(self):
+        self._test_get_ignored_traceback([0, 0, 0], None)
+
+    def test_single_member_trailing_chain(self):
+        self._test_get_ignored_traceback([0, 0, 1], 2)
+
+    def test_two_member_trailing_chain(self):
+        self._test_get_ignored_traceback([0, 1, 1], 1)
+
+    def test_first_traceback_ignored(self):
+        self._test_get_ignored_traceback([1, 0, 0], None)
+
+    def test_middle_traceback_ignored(self):
+        self._test_get_ignored_traceback([0, 1, 0], None)