Use testr for running functional tests and documentation

Alters tox.ini to use testr rather than nosetests.
Also, adds documentation on how to use tox to run a
single functional test class.

Change-Id: I1333ead397f338ee3da3d81b0e1de8e93283a2b5
Closes-Bug: #1490747

.testr.conf

@@ -1,7 +1,9 @@
 [DEFAULT]
 test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
              OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
-             ${PYTHON:-python} -m subunit.run discover -s ./barbican -t . $LISTOPT $IDOPTION
+             OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} \
+             ${PYTHON:-python} -m subunit.run discover -s ${OS_TEST_PATH:-./barbican} -t . $LISTOPT $IDOPTION
 
 test_id_option=--load-list $IDFILE
 test_list_option=--list
+group_regex=([^\.]+\.)+

doc/source/testing.rst

@@ -64,13 +64,22 @@ the functional tests through tox.
     tox -e functional
 
 
-By default, the functional tox job will use ``nosetests`` to execute the
-functional tests. This is primarily due to nose being a very well known and
-common workflow among developers. It is important to note that the gating
-job will actually use ``testr`` instead of ``nosetests``. If you discover
-issues while running your tests in the gate, then consider running ``testr``
-or :doc:`Devstack</setup/devstack>` to more closely replicate the gating
-environment.
+By default, the functional tox job will use ``testr`` to execute the
+functional tests as used in the gating job.
+
+.. note::
+
+    In order to run an individual functional test function, you must use the
+    following command:
+
+    .. code-block:: bash
+
+        # path starts inside functionaltests folder
+        tox -e functional -- path.to.test.file.class_name.function
+
+    Groups of tests can also be run with a regex match after the ``--``.
+    For more information on what can be done with ``testr``, please see:
+    http://testrepository.readthedocs.org/en/latest/MANUAL.html
 
 Remote Debugging
 ----------------
@@ -82,7 +91,7 @@ breakpoint in ``def on_post`` in ``barbican.api.controllers.secrets.py``
 will allow you to hit the breakpoint when a ``POST`` is done on the
 secrets URL.
 
-..note::
+.. note::
 
     After performing the ``POST`` the application will freeze. In order to use
     ``rpdb``, you must open up another terminal and run the following:

functionaltests/api/base.py

@@ -14,7 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 import abc
+import fixtures
 import logging
+import os
 import uuid
 
 import oslotest.base as oslotest
@@ -37,6 +39,9 @@ class TestCase(oslotest.BaseTestCase):
     max_sized_field = 'a' * max_field_size
     oversized_field = 'a' * (max_field_size + 1)
 
+    log_format = ('%(asctime)s %(process)d %(levelname)-8s '
+                  '[%(name)s] %(message)s')
+
     @classmethod
     def setUpClass(cls):
         cls.LOG = logging.getLogger(cls._get_full_case_name())
@@ -48,6 +53,20 @@ class TestCase(oslotest.BaseTestCase):
 
         self.client = client.BarbicanClient()
 
+        if (os.environ.get('OS_STDOUT_CAPTURE').lower() == 'true' or
+                os.environ.get('OS_STDOUT_CAPTURE') == '1'):
+            stdout = self.useFixture(fixtures.StringStream('stdout')).stream
+            self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
+        if (os.environ.get('OS_STDERR_CAPTURE').lower() == 'true' or
+                os.environ.get('OS_STDERR_CAPTURE') == '1'):
+            stderr = self.useFixture(fixtures.StringStream('stderr')).stream
+            self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
+        if (os.environ.get('OS_LOG_CAPTURE').lower() == 'true' or
+                os.environ.get('OS_LOG_CAPTURE') == '1'):
+            self.useFixture(fixtures.LoggerFixture(nuke_handlers=False,
+                                                   format=self.log_format,
+                                                   level=logging.DEBUG))
+
     def tearDown(self):
         super(TestCase, self).tearDown()
         self.LOG.info('Finished: %s\n', self._testMethodName)

functionaltests/post_test_hook.sh

@@ -18,4 +18,5 @@
 sudo pip install -r /opt/stack/new/barbican/test-requirements.txt
 
 cd /opt/stack/new/barbican/functionaltests
+echo 'Running Functional Tests'
 sudo ./run_tests.sh

functionaltests/pretty_tox.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -o pipefail
+
+TESTRARGS=$1
+python setup.py testr --testr-args="--subunit $TESTRARGS" | subunit-trace --no-failure-debug -f
+retval=$?
+echo -e "\nSlowest Tests:\n"
+testr slowest
+exit $retval

tox.ini

@@ -57,10 +57,10 @@ commands=
 [testenv:functional]
 # This tox env is purely to make local test development easier
 # Note: This requires local running instances of Barbican and Keystone
-deps =
-    {[testenv]deps}
-    nose
-commands = nosetests -v --processes=-1 --process-timeout=600 {toxinidir}/functionaltests {posargs}
+deps = -r{toxinidir}/test-requirements.txt
+setenv = OS_TEST_PATH={toxinidir}/functionaltests
+commands =
+    /bin/bash {toxinidir}/functionaltests/pretty_tox.sh '{posargs}'
 
 [flake8]
 # E711 ignored because of sqlalchemy override of == None