From e67ca803b9e4c53f3ce81fb835ca22a5df8f5fef Mon Sep 17 00:00:00 2001 From: Fernando Diaz Date: Fri, 28 Aug 2015 01:38:39 -0500 Subject: [PATCH] Use testr for running functional tests and documentation Alters tox.ini to use testr rather than nosetests. Also, adds documentation on how to use tox to run a single functional test class. Change-Id: I1333ead397f338ee3da3d81b0e1de8e93283a2b5 Closes-Bug: #1490747 --- .testr.conf | 4 +++- doc/source/testing.rst | 25 +++++++++++++++++-------- functionaltests/api/base.py | 19 +++++++++++++++++++ functionaltests/post_test_hook.sh | 1 + functionaltests/pretty_tox.sh | 10 ++++++++++ tox.ini | 8 ++++---- 6 files changed, 54 insertions(+), 13 deletions(-) create mode 100644 functionaltests/pretty_tox.sh diff --git a/.testr.conf b/.testr.conf index 03663a7b4..e7a868376 100644 --- a/.testr.conf +++ b/.testr.conf @@ -1,7 +1,9 @@ [DEFAULT] test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ - ${PYTHON:-python} -m subunit.run discover -s ./barbican -t . $LISTOPT $IDOPTION + OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} \ + ${PYTHON:-python} -m subunit.run discover -s ${OS_TEST_PATH:-./barbican} -t . $LISTOPT $IDOPTION test_id_option=--load-list $IDFILE test_list_option=--list +group_regex=([^\.]+\.)+ diff --git a/doc/source/testing.rst b/doc/source/testing.rst index 72ddaa297..444360a9f 100644 --- a/doc/source/testing.rst +++ b/doc/source/testing.rst @@ -64,13 +64,22 @@ the functional tests through tox. tox -e functional -By default, the functional tox job will use ``nosetests`` to execute the -functional tests. This is primarily due to nose being a very well known and -common workflow among developers. It is important to note that the gating -job will actually use ``testr`` instead of ``nosetests``. If you discover -issues while running your tests in the gate, then consider running ``testr`` -or :doc:`Devstack` to more closely replicate the gating -environment. +By default, the functional tox job will use ``testr`` to execute the +functional tests as used in the gating job. + +.. note:: + + In order to run an individual functional test function, you must use the + following command: + + .. code-block:: bash + + # path starts inside functionaltests folder + tox -e functional -- path.to.test.file.class_name.function + + Groups of tests can also be run with a regex match after the ``--``. + For more information on what can be done with ``testr``, please see: + http://testrepository.readthedocs.org/en/latest/MANUAL.html Remote Debugging ---------------- @@ -82,7 +91,7 @@ breakpoint in ``def on_post`` in ``barbican.api.controllers.secrets.py`` will allow you to hit the breakpoint when a ``POST`` is done on the secrets URL. -..note:: +.. note:: After performing the ``POST`` the application will freeze. In order to use ``rpdb``, you must open up another terminal and run the following: diff --git a/functionaltests/api/base.py b/functionaltests/api/base.py index 035d2ff73..979cd1406 100644 --- a/functionaltests/api/base.py +++ b/functionaltests/api/base.py @@ -14,7 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. """ import abc +import fixtures import logging +import os import uuid import oslotest.base as oslotest @@ -37,6 +39,9 @@ class TestCase(oslotest.BaseTestCase): max_sized_field = 'a' * max_field_size oversized_field = 'a' * (max_field_size + 1) + log_format = ('%(asctime)s %(process)d %(levelname)-8s ' + '[%(name)s] %(message)s') + @classmethod def setUpClass(cls): cls.LOG = logging.getLogger(cls._get_full_case_name()) @@ -48,6 +53,20 @@ class TestCase(oslotest.BaseTestCase): self.client = client.BarbicanClient() + if (os.environ.get('OS_STDOUT_CAPTURE').lower() == 'true' or + os.environ.get('OS_STDOUT_CAPTURE') == '1'): + stdout = self.useFixture(fixtures.StringStream('stdout')).stream + self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout)) + if (os.environ.get('OS_STDERR_CAPTURE').lower() == 'true' or + os.environ.get('OS_STDERR_CAPTURE') == '1'): + stderr = self.useFixture(fixtures.StringStream('stderr')).stream + self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr)) + if (os.environ.get('OS_LOG_CAPTURE').lower() == 'true' or + os.environ.get('OS_LOG_CAPTURE') == '1'): + self.useFixture(fixtures.LoggerFixture(nuke_handlers=False, + format=self.log_format, + level=logging.DEBUG)) + def tearDown(self): super(TestCase, self).tearDown() self.LOG.info('Finished: %s\n', self._testMethodName) diff --git a/functionaltests/post_test_hook.sh b/functionaltests/post_test_hook.sh index db5fe763a..472a3c3a5 100755 --- a/functionaltests/post_test_hook.sh +++ b/functionaltests/post_test_hook.sh @@ -18,4 +18,5 @@ sudo pip install -r /opt/stack/new/barbican/test-requirements.txt cd /opt/stack/new/barbican/functionaltests +echo 'Running Functional Tests' sudo ./run_tests.sh diff --git a/functionaltests/pretty_tox.sh b/functionaltests/pretty_tox.sh new file mode 100644 index 000000000..83df3a9d5 --- /dev/null +++ b/functionaltests/pretty_tox.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -o pipefail + +TESTRARGS=$1 +python setup.py testr --testr-args="--subunit $TESTRARGS" | subunit-trace --no-failure-debug -f +retval=$? +echo -e "\nSlowest Tests:\n" +testr slowest +exit $retval \ No newline at end of file diff --git a/tox.ini b/tox.ini index a4ee28020..3c34ff1f9 100644 --- a/tox.ini +++ b/tox.ini @@ -57,10 +57,10 @@ commands= [testenv:functional] # This tox env is purely to make local test development easier # Note: This requires local running instances of Barbican and Keystone -deps = - {[testenv]deps} - nose -commands = nosetests -v --processes=-1 --process-timeout=600 {toxinidir}/functionaltests {posargs} +deps = -r{toxinidir}/test-requirements.txt +setenv = OS_TEST_PATH={toxinidir}/functionaltests +commands = + /bin/bash {toxinidir}/functionaltests/pretty_tox.sh '{posargs}' [flake8] # E711 ignored because of sqlalchemy override of == None