Merge pull request #175 from cdent/inner-fixtures

Inner fixtures

.testr.conf

@@ -2,4 +2,4 @@
 test_command=${PYTHON:-python} -m subunit.run discover gabbi $LISTOPT $IDOPTION
 test_id_option=--load-list $IDFILE
 test_list_option=--list
-group_regex=(?:gabbi\.suitemaker\.(test_[^_]+_[^_]+)|tests\.test_intercept\.([^_]+))
+group_regex=(?:gabbi\.suitemaker\.(test_[^_]+_[^_]+)|tests\.test_(?:intercept|inner_fixture)\.([^_]+))

docs/source/fixtures.rst

@@ -3,7 +3,7 @@ Fixtures
 
 Each suite of tests is represented by a single YAML file, and may
 optionally use one or more fixtures to provide the necessary
-environment for tests to run.
+environment required by the tests in that file.
 
 Fixtures are implemented as nested context managers. Subclasses
 of :class:`~gabbi.fixture.GabbiFixture` must implement
@@ -37,3 +37,31 @@ about the exception will be stored on the fixture so that the
 ``stop_fixture`` method can decide if the exception should change how
 the fixture should clean up. The exception information can be found on
 ``exc_type``, ``exc_value`` and ``traceback`` method attributes.
+
+Inner Fixtures
+==============
+
+In some contexts (for example CI environments with a large
+number of tests being run in a broadly concurrent environment where
+output is logged to a single file) it can be important to capture and
+consolidate stray output that is produced during the tests and display
+it associated with an individual test. This can help debugging and
+avoids unusable output that is the result of multiple streams being
+interleaved.
+
+Inner fixtures have been added to support this. These are fixtures
+more in line with the tradtional ``unittest`` concept of fixtures:
+a class on which ``setUp`` and ``cleanUp`` is automatically called.
+
+:func:`~gabbi.driver.build_tests` accepts a named parameter
+arguments of ``inner_fixtures``. The value of that argument may be
+an ordered list of fixtures.Fixture_ classes that will be called
+when each individual test is set up.
+
+An example fixture that could be useful is the FakeLogger_.
+
+.. note:: At this time ``inner_fixtures`` are not supported when
+          using the pytest :doc:`loader <loader>`.
+
+.. _fixtures.Fixture: https://pypi.python.org/pypi/fixtures
+.. _FakeLogger: https://pypi.python.org/pypi/fixtures#fakelogger

gabbi/case.py

@@ -24,10 +24,10 @@ import os
 import re
 import sys
 import time
-import unittest
 from unittest import case
 from unittest import result
 
+import fixtures
 import six
 from six.moves import http_cookies
 from six.moves.urllib import parse as urlparse
@@ -93,7 +93,7 @@ def potentialFailure(func):
     return wrapper
 
 
-class HTTPTestCase(unittest.TestCase):
+class HTTPTestCase(fixtures.TestWithFixtures):
     """Encapsulate a single HTTP request as a TestCase.
 
     If the test is a member of a sequence of requests, ensure that prior
@@ -108,6 +108,8 @@ class HTTPTestCase(unittest.TestCase):
     def setUp(self):
         if not self.has_run:
             super(HTTPTestCase, self).setUp()
+            for fixture in self.inner_fixtures:
+                self.useFixture(fixture())
 
     def tearDown(self):
         if not self.has_run:

gabbi/driver.py

@@ -41,7 +41,8 @@ from gabbi import utils
 def build_tests(path, loader, host=None, port=8001, intercept=None,
                 test_loader_name=None, fixture_module=None,
                 response_handlers=None, content_handlers=None,
-                prefix='', require_ssl=False, url=None):
+                prefix='', require_ssl=False, url=None,
+                inner_fixtures=None):
     """Read YAML files from a directory to create tests.
 
     Each YAML file represents an ordered sequence of HTTP requests.
@@ -61,6 +62,9 @@ def build_tests(path, loader, host=None, port=8001, intercept=None,
     :param prefix: A URL prefix for all URLs that are not fully qualified.
     :param url: A full URL to test against. Replaces host, port and prefix.
     :param require_ssl: If ``True``, make all tests default to using SSL.
+    :param inner_fixtures: A list of ``Fixtures`` to use with each
+                           individual test request.
+    :type inner_fixtures: List of fixtures.Fixture classes.
     :rtype: TestSuite containing multiple TestSuites (one for each YAML file).
     """
 
@@ -116,7 +120,8 @@ def build_tests(path, loader, host=None, port=8001, intercept=None,
         file_suite = suitemaker.test_suite_from_dict(
             loader, test_base_name, suite_dict, path, host, port,
             fixture_module, intercept, prefix=prefix,
-            test_loader_name=test_loader_name, handlers=handler_objects)
+            test_loader_name=test_loader_name, handlers=handler_objects,
+            inner_fixtures=inner_fixtures)
         top_suite.addTest(file_suite)
     return top_suite
 

gabbi/suitemaker.py

@@ -37,7 +37,8 @@ class TestMaker(object):
 
     def __init__(self, test_base_name, test_defaults, test_directory,
                  fixture_classes, loader, host, port, intercept, prefix,
-                 response_handlers, content_handlers, test_loader_name=None):
+                 response_handlers, content_handlers, test_loader_name=None,
+                 inner_fixtures=None):
         self.test_base_name = test_base_name
         self.test_defaults = test_defaults
         self.default_keys = set(test_defaults.keys())
@@ -49,6 +50,7 @@ class TestMaker(object):
         self.intercept = intercept
         self.prefix = prefix
         self.test_loader_name = test_loader_name
+        self.inner_fixtures = inner_fixtures or []
         self.content_handlers = content_handlers
         self.response_handlers = response_handlers
 
@@ -85,6 +87,7 @@ class TestMaker(object):
                             {'test_data': test,
                              'test_directory': self.test_directory,
                              'fixtures': self.fixture_classes,
+                             'inner_fixtures': self.inner_fixtures,
                              'http': http_class,
                              'host': self.host,
                              'intercept': self.intercept,
@@ -168,7 +171,8 @@ class TestBuilder(type):
 
 def test_suite_from_dict(loader, test_base_name, suite_dict, test_directory,
                          host, port, fixture_module, intercept, prefix='',
-                         handlers=None, test_loader_name=None):
+                         handlers=None, test_loader_name=None,
+                         inner_fixtures=None):
     """Generate a GabbiSuite from a dict represent a list of tests.
 
     The dict takes the form:
@@ -216,7 +220,8 @@ def test_suite_from_dict(loader, test_base_name, suite_dict, test_directory,
     test_maker = TestMaker(test_base_name, default_test_dict, test_directory,
                            fixture_classes, loader, host, port, intercept,
                            prefix, response_handlers, content_handlers,
-                           test_loader_name)
+                           test_loader_name=test_loader_name,
+                           inner_fixtures=inner_fixtures)
     file_suite = suite.GabbiSuite()
     prior_test = None
     for test_dict in test_data:

gabbi/tests/gabbits_inner/inner.yaml

@@ -0,0 +1,14 @@
+
+fixtures:
+    - OuterFixture
+
+tests:
+
+- name: get one
+  GET: /
+
+- name: get two
+  GET: /
+
+- name: get three
+  GET: /

gabbi/tests/test_inner_fixture.py

@@ -0,0 +1,63 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Test the works of inner and outer fixtures.
+
+An "outer" fixture runs once per test suite. An "inner" is per test request.
+"""
+
+import os
+import sys
+
+import fixtures
+
+from gabbi import driver
+from gabbi import fixture
+from gabbi.tests import simple_wsgi
+
+
+TESTS_DIR = 'gabbits_inner'
+COUNT_INNER = 0
+COUNT_OUTER = 0
+
+
+class OuterFixture(fixture.GabbiFixture):
+    """Assert an outer fixture is only started once and is stopped."""
+
+    def start_fixture(self):
+        global COUNT_OUTER
+        COUNT_OUTER += 1
+
+    def stop_fixture(self):
+        assert COUNT_OUTER == 1
+
+
+class InnerFixture(fixtures.Fixture):
+    """Test that setUp is called 3 times."""
+
+    def setUp(self):
+        super(InnerFixture, self).setUp()
+        global COUNT_INNER
+        COUNT_INNER += 1
+
+    def cleanUp(self):
+        super(InnerFixture, self).cleanUp()
+        assert 1 <= COUNT_INNER <= 3
+
+
+def load_tests(loader, tests, pattern):
+    test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
+    return driver.build_tests(test_dir, loader, host=None,
+                              intercept=simple_wsgi.SimpleWsgi,
+                              fixture_module=sys.modules[__name__],
+                              inner_fixtures=[InnerFixture],
+                              test_loader_name=__name__)

gabbi/tests/test_runner.py

@@ -13,7 +13,6 @@
 """Test that the CLI works as expected
 """
 
-import copy
 import sys
 import unittest
 from uuid import uuid4
@@ -21,7 +20,6 @@ from uuid import uuid4
 from six import StringIO
 from wsgi_intercept.interceptor import Urllib3Interceptor
 
-from gabbi import case
 from gabbi import exception
 from gabbi.handlers import base
 from gabbi import runner
@@ -57,9 +55,6 @@ class RunnerTest(unittest.TestCase):
         sys.stdout = self._stdout
         sys.stderr = self._stderr
         sys.argv = self._argv
-        # Cleanup the custom response_handler
-        case.HTTPTestCase.response_handlers = []
-        case.HTTPTestCase.base_test = copy.copy(case.BASE_TEST)
 
     def test_target_url_parsing(self):
         sys.argv = ['gabbi-run', 'http://%s:%s/foo' % (self.host, self.port)]

requirements.txt

@@ -6,3 +6,4 @@ urllib3>=1.11.0
 jsonpath-rw-ext>=1.0.0
 wsgi-intercept>=1.2.2
 colorama
+fixtures