openstack/rally
Code Issues Proposed changes

[Verify] Fixing 'xfail' mechanism and whole test case failure issue

Browse Source 
If Tempest fails in a test's setUpClass(), the result that will be reported
by Rally is "fail", even though the test is listed in a file passed to
--xfails-file. So this commit fixes the issue.

Also, this commit adds missing unittest for the subunit parser when we have
a file with expected failures.

Closes-Bug: #1568133

Change-Id: I9e10891f02387b45a507a02326ecf81f0cb97bf3
This commit is contained in:
Yaroslav Lobankov 2016-04-12 14:30:47 +03:00
parent ffe91c9c07
commit a26bc6b2c5
2 changed files with 33 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
rally/common/io/subunit_v2.py
View File

@ -81,9 +81,9 @@ class SubunitV2StreamResult(object):
def _post_parse(self):
# parse unknown entities
for test_id in self._unknown_entities:
# NOTE(andreykurilin): When whole TestCase is marked as skipped,
# there is only one event with reason and status, so we should
# modify all tests of TestCase manually.
# NOTE(andreykurilin): When whole TestCase is marked as skipped
# or failed, there is only one event with reason and status, so
# we should modify all tests of TestCase manually.
matcher = lambda i: i == test_id or i.startswith("%s." % test_id)
known_ids = filter(matcher, self._tests)
for id_ in known_ids:
@ -97,6 +97,17 @@ class SubunitV2StreamResult(object):
self._tests[id_]["traceback"] = (
self._unknown_entities[test_id]["traceback"])
# parse expected failures
for test_id in self._expected_failures:
if self._tests.get(test_id):
if self._tests[test_id]["status"] == "fail":
self._tests[test_id]["status"] = "xfail"
if self._expected_failures[test_id]:
self._tests[test_id]["reason"] = (
self._expected_failures[test_id])
elif self._tests[test_id]["status"] == "success":
self._tests[test_id]["status"] = "uxsuccess"
# decode data
for test_id in self._tests:
for file_name in ["traceback", "reason"]:
@ -105,6 +116,7 @@ class SubunitV2StreamResult(object):
self._tests[test_id][file_name] = (
encodeutils.safe_decode(
self._tests[test_id][file_name]))
self._is_parsed = True
@property
@ -145,14 +157,6 @@ class SubunitV2StreamResult(object):
elif test_status:
self._tests[test_id]["time"] = total_seconds(
timestamp - self._timestamps[test_id])
if test_id in self._expected_failures:
if test_status == "fail":
test_status = "xfail"
if self._expected_failures[test_id]:
self._tests[test_id]["reason"] = (
self._expected_failures[test_id])
elif test_status == "success":
test_status = "uxsuccess"
self._tests[test_id]["status"] = test_status
else:
if file_name in ["traceback", "reason"]:

17
tests/unit/common/io/test_subunit_v2.py
View File

@ -114,3 +114,20 @@ RuntimeError: broken setUp method
"success": 0, "unexpected_success": 0,
"expected_failures": 0},
subunit_v2.SubunitV2StreamResult().total)
def test_parse_results_file_with_expected_failures(self):
test_1 = "test_foo.SimpleTestCase.test_something_that_fails"
test_2 = "test_foo.SimpleTestCase.test_something_that_passes"
expected_failures = {
test_1: "Some details about why this test fails",
test_2: None
}
result = subunit_v2.parse_results_file(self.fake_stream,
expected_failures)
tests = result.tests
self.assertEqual("xfail", tests[test_1]["status"])
self.assertEqual("Some details about why this test fails",
tests[test_1]["reason"])
self.assertEqual("uxsuccess", tests[test_2]["status"])