From a26bc6b2c5b2b9083019292c126416a215391083 Mon Sep 17 00:00:00 2001 From: Yaroslav Lobankov Date: Tue, 12 Apr 2016 14:30:47 +0300 Subject: [PATCH] [Verify] Fixing 'xfail' mechanism and whole test case failure issue If Tempest fails in a test's setUpClass(), the result that will be reported by Rally is "fail", even though the test is listed in a file passed to --xfails-file. So this commit fixes the issue. Also, this commit adds missing unittest for the subunit parser when we have a file with expected failures. Closes-Bug: #1568133 Change-Id: I9e10891f02387b45a507a02326ecf81f0cb97bf3 --- rally/common/io/subunit_v2.py | 26 ++++++++++++++----------- tests/unit/common/io/test_subunit_v2.py | 19 +++++++++++++++++- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/rally/common/io/subunit_v2.py b/rally/common/io/subunit_v2.py index ed68bfce3d..04b57c9da4 100644 --- a/rally/common/io/subunit_v2.py +++ b/rally/common/io/subunit_v2.py @@ -81,9 +81,9 @@ class SubunitV2StreamResult(object): def _post_parse(self): # parse unknown entities for test_id in self._unknown_entities: - # NOTE(andreykurilin): When whole TestCase is marked as skipped, - # there is only one event with reason and status, so we should - # modify all tests of TestCase manually. + # NOTE(andreykurilin): When whole TestCase is marked as skipped + # or failed, there is only one event with reason and status, so + # we should modify all tests of TestCase manually. matcher = lambda i: i == test_id or i.startswith("%s." % test_id) known_ids = filter(matcher, self._tests) for id_ in known_ids: @@ -97,6 +97,17 @@ class SubunitV2StreamResult(object): self._tests[id_]["traceback"] = ( self._unknown_entities[test_id]["traceback"]) + # parse expected failures + for test_id in self._expected_failures: + if self._tests.get(test_id): + if self._tests[test_id]["status"] == "fail": + self._tests[test_id]["status"] = "xfail" + if self._expected_failures[test_id]: + self._tests[test_id]["reason"] = ( + self._expected_failures[test_id]) + elif self._tests[test_id]["status"] == "success": + self._tests[test_id]["status"] = "uxsuccess" + # decode data for test_id in self._tests: for file_name in ["traceback", "reason"]: @@ -105,6 +116,7 @@ class SubunitV2StreamResult(object): self._tests[test_id][file_name] = ( encodeutils.safe_decode( self._tests[test_id][file_name])) + self._is_parsed = True @property @@ -145,14 +157,6 @@ class SubunitV2StreamResult(object): elif test_status: self._tests[test_id]["time"] = total_seconds( timestamp - self._timestamps[test_id]) - if test_id in self._expected_failures: - if test_status == "fail": - test_status = "xfail" - if self._expected_failures[test_id]: - self._tests[test_id]["reason"] = ( - self._expected_failures[test_id]) - elif test_status == "success": - test_status = "uxsuccess" self._tests[test_id]["status"] = test_status else: if file_name in ["traceback", "reason"]: diff --git a/tests/unit/common/io/test_subunit_v2.py b/tests/unit/common/io/test_subunit_v2.py index b6efb986ea..774b4e4f69 100644 --- a/tests/unit/common/io/test_subunit_v2.py +++ b/tests/unit/common/io/test_subunit_v2.py @@ -113,4 +113,21 @@ RuntimeError: broken setUp method self.assertEqual({"tests": 0, "time": 0, "failures": 0, "skipped": 0, "success": 0, "unexpected_success": 0, "expected_failures": 0}, - subunit_v2.SubunitV2StreamResult().total) \ No newline at end of file + subunit_v2.SubunitV2StreamResult().total) + + def test_parse_results_file_with_expected_failures(self): + test_1 = "test_foo.SimpleTestCase.test_something_that_fails" + test_2 = "test_foo.SimpleTestCase.test_something_that_passes" + expected_failures = { + test_1: "Some details about why this test fails", + test_2: None + } + + result = subunit_v2.parse_results_file(self.fake_stream, + expected_failures) + tests = result.tests + + self.assertEqual("xfail", tests[test_1]["status"]) + self.assertEqual("Some details about why this test fails", + tests[test_1]["reason"]) + self.assertEqual("uxsuccess", tests[test_2]["status"])