openstack/deb-python-gabbi
Code Issues Proposed changes

Adjust runner to track test suite failures

Browse Source 
Change the reporter used by gabbi-run so that it provides a more
complete test name in its output, using the id() of the test. This
will include the name of the yaml file.

Also track failures per input file and provide a summary of which
files contained failures (if any) at the end.

Both of these changes were done in the simplest way possible so
there is probably considerably room for improvement now that the
concept has been proven.

Fixes #181
This commit is contained in:
Chris Dent 2017-02-02 17:49:27 +00:00
parent fb9a001db7
commit f0bacce539
2 changed files with 16 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
gabbi/reporter.py
View File

@ -80,7 +80,8 @@ class ConciseTestResult(TextTestResult):
self.stream.writeln('\t[unexpected success]')
def getDescription(self, test):
name = test.test_data['name']
# Chop the test method (test_request) off the test.id().
name = test.id().rsplit('.', 1)[0]
desc = test.test_data.get('desc', None)
return ': '.join((name, desc)) if desc else name

18
gabbi/runner.py
View File

@ -12,6 +12,8 @@
# under the License.
"""Implementation of a command-line runner for gabbi files (AKA suites)."""
from __future__ import print_function
import argparse
from importlib import import_module
import os
@ -81,6 +83,8 @@ def run():
verbosity = args.verbosity
failfast = args.failfast
failure = False
# Keep track of file names that have failures.
failures = []
if not input_files:
success = run_suite(sys.stdin, handler_objects, host, port,
@ -89,22 +93,28 @@ def run():
failure = not success
else:
for input_file in input_files:
name = os.path.splitext(os.path.basename(input_file))[0]
with open(input_file, 'r') as fh:
data_dir = os.path.dirname(input_file)
success = run_suite(fh, handler_objects, host, port,
prefix, force_ssl, failfast,
data_dir=data_dir,
verbosity=verbosity)
verbosity=verbosity, name=name)
if not success:
failures.append(input_file)
if not failure: # once failed, this is considered immutable
failure = not success
if failure and failfast:
break
if failures:
print("There were failures in the following files:", file=sys.stderr)
print('\n'.join(failures), file=sys.stderr)
sys.exit(failure)
def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False,
failfast=False, data_dir='.', verbosity=False):
failfast=False, data_dir='.', verbosity=False, name='input'):
"""Run the tests from the YAML in handle."""
data = utils.load_yaml(handle)
if force_ssl:
@ -120,8 +130,8 @@ def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False,
loader = unittest.defaultTestLoader
test_suite = suitemaker.test_suite_from_dict(
loader, 'input', data, data_dir, host, port, None, None, prefix=prefix,
handlers=handler_objects)
loader, name, data, data_dir, host, port, None, None, prefix=prefix,
handlers=handler_objects, test_loader_name='gabbi-runner')
result = ConciseTestRunner(
verbosity=2, failfast=failfast).run(test_suite)