diff --git a/.testr.conf b/.testr.conf index 0c24566..eba67ce 100644 --- a/.testr.conf +++ b/.testr.conf @@ -2,4 +2,4 @@ test_command=${PYTHON:-python} -m subunit.run discover gabbi $LISTOPT $IDOPTION test_id_option=--load-list $IDFILE test_list_option=--list -group_regex=(?:gabbi\.suitemaker\.(test_[^_]+_[^_]+)|tests\.test_intercept\.([^_]+)) +group_regex=(?:gabbi\.suitemaker\.(test_[^_]+_[^_]+)|tests\.test_(?:intercept|inner_fixture)\.([^_]+)) diff --git a/docs/source/fixtures.rst b/docs/source/fixtures.rst index c8ffdd3..52cb85a 100644 --- a/docs/source/fixtures.rst +++ b/docs/source/fixtures.rst @@ -3,7 +3,7 @@ Fixtures Each suite of tests is represented by a single YAML file, and may optionally use one or more fixtures to provide the necessary -environment for tests to run. +environment required by the tests in that file. Fixtures are implemented as nested context managers. Subclasses of :class:`~gabbi.fixture.GabbiFixture` must implement @@ -37,3 +37,31 @@ about the exception will be stored on the fixture so that the ``stop_fixture`` method can decide if the exception should change how the fixture should clean up. The exception information can be found on ``exc_type``, ``exc_value`` and ``traceback`` method attributes. + +Inner Fixtures +============== + +In some contexts (for example CI environments with a large +number of tests being run in a broadly concurrent environment where +output is logged to a single file) it can be important to capture and +consolidate stray output that is produced during the tests and display +it associated with an individual test. This can help debugging and +avoids unusable output that is the result of multiple streams being +interleaved. + +Inner fixtures have been added to support this. These are fixtures +more in line with the tradtional ``unittest`` concept of fixtures: +a class on which ``setUp`` and ``cleanUp`` is automatically called. + +:func:`~gabbi.driver.build_tests` accepts a named parameter +arguments of ``inner_fixtures``. The value of that argument may be +an ordered list of fixtures.Fixture_ classes that will be called +when each individual test is set up. + +An example fixture that could be useful is the FakeLogger_. + +.. note:: At this time ``inner_fixtures`` are not supported when + using the pytest :doc:`loader `. + +.. _fixtures.Fixture: https://pypi.python.org/pypi/fixtures +.. _FakeLogger: https://pypi.python.org/pypi/fixtures#fakelogger diff --git a/gabbi/case.py b/gabbi/case.py index c5a1674..58bc5d0 100644 --- a/gabbi/case.py +++ b/gabbi/case.py @@ -24,10 +24,10 @@ import os import re import sys import time -import unittest from unittest import case from unittest import result +import fixtures import six from six.moves import http_cookies from six.moves.urllib import parse as urlparse @@ -93,7 +93,7 @@ def potentialFailure(func): return wrapper -class HTTPTestCase(unittest.TestCase): +class HTTPTestCase(fixtures.TestWithFixtures): """Encapsulate a single HTTP request as a TestCase. If the test is a member of a sequence of requests, ensure that prior @@ -108,6 +108,8 @@ class HTTPTestCase(unittest.TestCase): def setUp(self): if not self.has_run: super(HTTPTestCase, self).setUp() + for fixture in self.inner_fixtures: + self.useFixture(fixture()) def tearDown(self): if not self.has_run: diff --git a/gabbi/driver.py b/gabbi/driver.py index 8a28425..77afe7e 100644 --- a/gabbi/driver.py +++ b/gabbi/driver.py @@ -41,7 +41,8 @@ from gabbi import utils def build_tests(path, loader, host=None, port=8001, intercept=None, test_loader_name=None, fixture_module=None, response_handlers=None, content_handlers=None, - prefix='', require_ssl=False, url=None): + prefix='', require_ssl=False, url=None, + inner_fixtures=None): """Read YAML files from a directory to create tests. Each YAML file represents an ordered sequence of HTTP requests. @@ -61,6 +62,9 @@ def build_tests(path, loader, host=None, port=8001, intercept=None, :param prefix: A URL prefix for all URLs that are not fully qualified. :param url: A full URL to test against. Replaces host, port and prefix. :param require_ssl: If ``True``, make all tests default to using SSL. + :param inner_fixtures: A list of ``Fixtures`` to use with each + individual test request. + :type inner_fixtures: List of fixtures.Fixture classes. :rtype: TestSuite containing multiple TestSuites (one for each YAML file). """ @@ -116,7 +120,8 @@ def build_tests(path, loader, host=None, port=8001, intercept=None, file_suite = suitemaker.test_suite_from_dict( loader, test_base_name, suite_dict, path, host, port, fixture_module, intercept, prefix=prefix, - test_loader_name=test_loader_name, handlers=handler_objects) + test_loader_name=test_loader_name, handlers=handler_objects, + inner_fixtures=inner_fixtures) top_suite.addTest(file_suite) return top_suite diff --git a/gabbi/suitemaker.py b/gabbi/suitemaker.py index 392d4c9..7f16030 100644 --- a/gabbi/suitemaker.py +++ b/gabbi/suitemaker.py @@ -37,7 +37,8 @@ class TestMaker(object): def __init__(self, test_base_name, test_defaults, test_directory, fixture_classes, loader, host, port, intercept, prefix, - response_handlers, content_handlers, test_loader_name=None): + response_handlers, content_handlers, test_loader_name=None, + inner_fixtures=None): self.test_base_name = test_base_name self.test_defaults = test_defaults self.default_keys = set(test_defaults.keys()) @@ -49,6 +50,7 @@ class TestMaker(object): self.intercept = intercept self.prefix = prefix self.test_loader_name = test_loader_name + self.inner_fixtures = inner_fixtures or [] self.content_handlers = content_handlers self.response_handlers = response_handlers @@ -85,6 +87,7 @@ class TestMaker(object): {'test_data': test, 'test_directory': self.test_directory, 'fixtures': self.fixture_classes, + 'inner_fixtures': self.inner_fixtures, 'http': http_class, 'host': self.host, 'intercept': self.intercept, @@ -168,7 +171,8 @@ class TestBuilder(type): def test_suite_from_dict(loader, test_base_name, suite_dict, test_directory, host, port, fixture_module, intercept, prefix='', - handlers=None, test_loader_name=None): + handlers=None, test_loader_name=None, + inner_fixtures=None): """Generate a GabbiSuite from a dict represent a list of tests. The dict takes the form: @@ -216,7 +220,8 @@ def test_suite_from_dict(loader, test_base_name, suite_dict, test_directory, test_maker = TestMaker(test_base_name, default_test_dict, test_directory, fixture_classes, loader, host, port, intercept, prefix, response_handlers, content_handlers, - test_loader_name) + test_loader_name=test_loader_name, + inner_fixtures=inner_fixtures) file_suite = suite.GabbiSuite() prior_test = None for test_dict in test_data: diff --git a/gabbi/tests/gabbits_inner/inner.yaml b/gabbi/tests/gabbits_inner/inner.yaml new file mode 100644 index 0000000..c25fb70 --- /dev/null +++ b/gabbi/tests/gabbits_inner/inner.yaml @@ -0,0 +1,14 @@ + +fixtures: + - OuterFixture + +tests: + +- name: get one + GET: / + +- name: get two + GET: / + +- name: get three + GET: / diff --git a/gabbi/tests/test_inner_fixture.py b/gabbi/tests/test_inner_fixture.py new file mode 100644 index 0000000..6a3f1ac --- /dev/null +++ b/gabbi/tests/test_inner_fixture.py @@ -0,0 +1,63 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Test the works of inner and outer fixtures. + +An "outer" fixture runs once per test suite. An "inner" is per test request. +""" + +import os +import sys + +import fixtures + +from gabbi import driver +from gabbi import fixture +from gabbi.tests import simple_wsgi + + +TESTS_DIR = 'gabbits_inner' +COUNT_INNER = 0 +COUNT_OUTER = 0 + + +class OuterFixture(fixture.GabbiFixture): + """Assert an outer fixture is only started once and is stopped.""" + + def start_fixture(self): + global COUNT_OUTER + COUNT_OUTER += 1 + + def stop_fixture(self): + assert COUNT_OUTER == 1 + + +class InnerFixture(fixtures.Fixture): + """Test that setUp is called 3 times.""" + + def setUp(self): + super(InnerFixture, self).setUp() + global COUNT_INNER + COUNT_INNER += 1 + + def cleanUp(self): + super(InnerFixture, self).cleanUp() + assert 1 <= COUNT_INNER <= 3 + + +def load_tests(loader, tests, pattern): + test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR) + return driver.build_tests(test_dir, loader, host=None, + intercept=simple_wsgi.SimpleWsgi, + fixture_module=sys.modules[__name__], + inner_fixtures=[InnerFixture], + test_loader_name=__name__) diff --git a/gabbi/tests/test_runner.py b/gabbi/tests/test_runner.py index 1c5fc5b..503cfe0 100644 --- a/gabbi/tests/test_runner.py +++ b/gabbi/tests/test_runner.py @@ -13,7 +13,6 @@ """Test that the CLI works as expected """ -import copy import sys import unittest from uuid import uuid4 @@ -21,7 +20,6 @@ from uuid import uuid4 from six import StringIO from wsgi_intercept.interceptor import Urllib3Interceptor -from gabbi import case from gabbi import exception from gabbi.handlers import base from gabbi import runner @@ -57,9 +55,6 @@ class RunnerTest(unittest.TestCase): sys.stdout = self._stdout sys.stderr = self._stderr sys.argv = self._argv - # Cleanup the custom response_handler - case.HTTPTestCase.response_handlers = [] - case.HTTPTestCase.base_test = copy.copy(case.BASE_TEST) def test_target_url_parsing(self): sys.argv = ['gabbi-run', 'http://%s:%s/foo' % (self.host, self.port)] diff --git a/requirements.txt b/requirements.txt index a3fae2b..b292fce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ urllib3>=1.11.0 jsonpath-rw-ext>=1.0.0 wsgi-intercept>=1.2.2 colorama +fixtures