openstack
/
os-testr
Code Issues Proposed changes

Sort failed tests at the top on HTML result page 

Currently tests are sorted alphabetically by test class name regardless
of the pass/fail status. When troubleshooting failed unit tests in
jenkins jobs with many unit tests (> 11242 in nova), one must scroll
potentially a lot or wait as the page loads to search for the word
'fail' to get to the details of the failed test.

This sorts failed (and error) tests at the top of the HTML page for
easier troubleshooting (sorted failed tests + sorted passed tests).

Change-Id: I0b575e77c4a3e1cc73e60ea48ca5b9bf2e84d76f
This commit is contained in:
melanie witt 2016-05-26 20:31:30 +00:00
parent e5e11dca2e
commit f95f1d4448
2 changed files with 53 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
os_testr/subunit2html.py
View File

@ -617,8 +617,12 @@ class HtmlOutput(testtools.TestResult):
# unittest does not seems to run in any particular order.
# Here at least we want to group them together by class.
rmap = {}
classes = []
# Differentiate between classes that have test failures so we can sort
# them at the top of the html page for easier troubleshooting
failclasses = []
passclasses = []
for n, t, o, e in result_list:
classes = failclasses if n == 1 or n == 2 else passclasses
if hasattr(t, '_tests'):
for inner_test in t._tests:
self._add_cls(rmap, classes, inner_test,
@ -626,7 +630,9 @@ class HtmlOutput(testtools.TestResult):
else:
self._add_cls(rmap, classes, t, (n, t, o, e))
classort = lambda s: str(s)
sortedclasses = sorted(classes, key=classort)
sortedfailclasses = sorted(failclasses, key=classort)
sortedpassclasses = sorted(passclasses, key=classort)
sortedclasses = sortedfailclasses + sortedpassclasses
r = [(cls, rmap[str(cls)]) for cls in sortedclasses]
return r

45
os_testr/tests/test_subunit2html.py
View File

@ -10,6 +10,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import sys
from ddt import data
from ddt import ddt
from subunit import RemotedTestCase
@ -29,3 +31,46 @@ class TestSubunit2html(base.TestCase):
cls_ = []
obj_._add_cls({}, cls_, test_, ())
self.assertEqual("example.path.to.test", cls_[0].name)
@data(RemotedTestCase, PlaceHolder)
def test_result_sorting(self, test_cls):
tests = []
for i in range(9):
tests.append(test_cls('example.path.to.test%d.method' % i))
# addFailure, addError, and addSkip need the real exc_info
try:
raise Exception('fake')
except Exception:
err = sys.exc_info()
obj = subunit2html.HtmlOutput()
obj.addSuccess(tests[3])
obj.addSuccess(tests[1])
# example.path.to.test2 has a failure
obj.addFailure(tests[2], err)
obj.addSkip(tests[0], err)
obj.addSuccess(tests[8])
# example.path.to.test5 has a failure (error)
obj.addError(tests[5], err)
# example.path.to.test4 has a failure
obj.addFailure(tests[4], err)
obj.addSuccess(tests[7])
# example.path.to.test6 has a failure
obj.addFailure(tests[6], err)
sorted_result = obj._sortResult(obj.result)
# _sortResult returns a list of results of format:
# [(class, [test_result_tuple, ...]), ...]
# sorted by str(class)
#
# Classes with failures (2, 4, 5, and 6) should be sorted separately
# at the top. The rest of the classes should be in sorted order after.
expected_class_order = ['example.path.to.test2',
'example.path.to.test4',
'example.path.to.test5',
'example.path.to.test6',
'example.path.to.test0',
'example.path.to.test1',
'example.path.to.test3',
'example.path.to.test7',
'example.path.to.test8']
for i, r in enumerate(sorted_result):
self.assertEqual(expected_class_order[i], str(r[0]))