From 384fbf4977756fa9a9b306d7a0b1f9b6d34c9848 Mon Sep 17 00:00:00 2001 From: Andrey Kurilin Date: Fri, 28 Feb 2020 18:23:55 +0200 Subject: [PATCH] Rework inner code for JUnit reports Now we have unified code for both task exporter and verification reporter. Also, difference in attributes order between python versions are fixed. Covers https://bugs.python.org/issue34160 Change-Id: I9a86e995d2ecb78a3ec9a69eaa5815296fcb4a6e --- rally/common/io/junit.py | 185 +++++++++++++----- rally/common/utils.py | 22 +-- rally/plugins/common/exporters/junit.py | 61 ++---- .../plugins/common/verification/reporters.py | 78 +++----- rally/plugins/task/__init__.py | 0 rally/plugins/verification/__init__.py | 0 tests/ci/playbooks/rally-install/pre.yaml | 1 + tests/unit/common/io/test_junit.py | 59 +++--- .../plugins/common/exporters/test_junit.py | 12 +- .../common/verification/test_reporters.py | 12 +- 10 files changed, 229 insertions(+), 201 deletions(-) create mode 100644 rally/plugins/task/__init__.py create mode 100644 rally/plugins/verification/__init__.py diff --git a/rally/common/io/junit.py b/rally/common/io/junit.py index 4245b888a0..21582b7a9e 100644 --- a/rally/common/io/junit.py +++ b/rally/common/io/junit.py @@ -1,4 +1,3 @@ -# Copyright 2015: eNovance # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -13,56 +12,150 @@ # License for the specific language governing permissions and limitations # under the License. +import collections +import datetime as dt import xml.etree.ElementTree as ET +from rally.common import version -class JUnit(object): - SUCCESS = "success" - FAILURE = "failure" - ERROR = "error" - def __init__(self, test_suite_name): - self.test_suite_name = test_suite_name - self.test_cases = [] - self.n_tests = 0 - self.n_failures = 0 - self.n_errors = 0 - self.total_time = 0.0 +def _prettify_xml(elem, level=0): + """Adds indents. - def add_test(self, test_name, time, outcome=SUCCESS, message=""): - class_name, name = test_name.split(".", 1) - self.test_cases.append({ - "classname": class_name, - "name": name, - "time": str("%.2f" % time), - "outcome": outcome, - "message": message - }) + Code of this method was copied from + http://effbot.org/zone/element-lib.htm#prettyprint - if outcome == JUnit.FAILURE: - self.n_failures += 1 - elif outcome == JUnit.ERROR: - self.n_errors += 1 - elif outcome != JUnit.SUCCESS: - raise ValueError("Unexpected outcome %s" % outcome) + """ + i = "\n" + level * " " + if len(elem): + if not elem.text or not elem.text.strip(): + elem.text = i + " " + if not elem.tail or not elem.tail.strip(): + elem.tail = i + for elem in elem: + _prettify_xml(elem, level + 1) + if not elem.tail or not elem.tail.strip(): + elem.tail = i + else: + if level and (not elem.tail or not elem.tail.strip()): + elem.tail = i - self.n_tests += 1 - self.total_time += time - def to_xml(self): - xml = ET.Element("testsuite", { - "name": self.test_suite_name, - "tests": str(self.n_tests), - "time": str("%.2f" % self.total_time), - "failures": str(self.n_failures), - "errors": str(self.n_errors), - }) - for test_case in self.test_cases: - outcome = test_case.pop("outcome") - message = test_case.pop("message") - if outcome in [JUnit.FAILURE, JUnit.ERROR]: - sub = ET.SubElement(xml, "testcase", test_case) - sub.append(ET.Element(outcome, {"message": message})) - else: - xml.append(ET.Element("testcase", test_case)) - return ET.tostring(xml, encoding="utf-8").decode("utf-8") +def _filter_attrs(**attrs): + return collections.OrderedDict( + (k, v) for k, v in sorted(attrs.items()) if v is not None) + + +class _TestCase(object): + def __init__(self, parent, classname, name, id=None, time=None, + timestamp=None): + self._parent = parent + attrs = _filter_attrs(id=id, time=time, classname=classname, + name=name, timestamp=timestamp) + self._elem = ET.SubElement(self._parent._elem, "testcase", **attrs) + + def _add_details(self, tag=None, text=None, *comments): + if tag: + elem = ET.SubElement(self._elem, tag) + if text: + elem.text = text + for comment in comments: + if comment: + self._elem.append(ET.Comment(comment)) + + def mark_as_failed(self, details): + self._add_details("failure", details) + self._parent._increment("failures") + + def mark_as_uxsuccess(self, reason=None): + # NOTE(andreykurilin): junit doesn't support uxsuccess + # status, so let's display it like "fail" with proper comment. + self.mark_as_failed( + f"It is an unexpected success. The test " + f"should fail due to: {reason or 'Unknown reason'}" + ) + + def mark_as_xfail(self, reason=None, details=None): + reason = (f"It is an expected failure due to: " + f"{reason or 'Unknown reason'}") + self._add_details(None, None, reason, details) + + def mark_as_skipped(self, reason): + self._add_details("skipped", reason or "Unknown reason") + self._parent._increment("skipped") + + +class _TestSuite(object): + def __init__(self, parent, id, time, timestamp): + self._parent = parent + attrs = _filter_attrs(id=id, time=time, tests="0", + errors="0", skipped="0", + failures="0", timestamp=timestamp) + self._elem = ET.SubElement(self._parent, "testsuite", **attrs) + + self._finalized = False + self._calculate = True + self._total = 0 + self._skipped = 0 + self._failures = 0 + + def _finalize(self): + if not self._finalized and self._calculate: + self._setup_final_stats(tests=str(self._total), + skipped=str(self._skipped), + failures=str(self._failures)) + self._finalized = True + + def _setup_final_stats(self, tests, skipped, failures): + self._elem.set("tests", tests) + self._elem.set("skipped", skipped) + self._elem.set("failures", failures) + + def setup_final_stats(self, tests, skipped, failures): + """Turn off calculation of final stats.""" + self._calculate = False + self._setup_final_stats(tests, skipped, failures) + + def _increment(self, status): + if self._calculate: + key = f"_{status}" + value = getattr(self, key) + 1 + setattr(self, key, value) + self._finalized = False + + def add_test_case(self, classname, name, id=None, time=None, + timestamp=None): + self._increment("total") + return _TestCase(self, id=id, classname=classname, name=name, + time=time, timestamp=timestamp) + + +class JUnitXML(object): + """A helper class to build JUnit-XML report without knowing XML.""" + + def __init__(self): + self._root = ET.Element("testsuites") + self._test_suites = [] + + self._root.append( + ET.Comment("Report is generated by Rally %s at %s" % ( + version.version_string(), + dt.datetime.utcnow().isoformat())) + ) + + def __str__(self): + return self.to_string() + + def to_string(self): + for test_suite in self._test_suites: + test_suite._finalize() + + _prettify_xml(self._root) + + return ET.tostring(self._root, encoding="utf-8").decode("utf-8") + + def add_test_suite(self, id, time, timestamp): + test_suite = _TestSuite( + self._root, id=id, time=time, timestamp=timestamp) + self._test_suites.append(test_suite) + return test_suite diff --git a/rally/common/utils.py b/rally/common/utils.py index df3d535344..35483c35f5 100644 --- a/rally/common/utils.py +++ b/rally/common/utils.py @@ -32,6 +32,7 @@ import uuid from six import moves +from rally.common.io import junit from rally.common import logging from rally import exceptions @@ -786,23 +787,6 @@ class BackupHelper(object): shutil.rmtree(path) +@logging.log_deprecated("it was an inner helper.", rally_version="3.0.0") def prettify_xml(elem, level=0): - """Adds indents. - - Code of this method was copied from - http://effbot.org/zone/element-lib.htm#prettyprint - - """ - i = "\n" + level * " " - if len(elem): - if not elem.text or not elem.text.strip(): - elem.text = i + " " - if not elem.tail or not elem.tail.strip(): - elem.tail = i - for elem in elem: - prettify_xml(elem, level + 1) - if not elem.tail or not elem.tail.strip(): - elem.tail = i - else: - if level and (not elem.tail or not elem.tail.strip()): - elem.tail = i + return junit._prettify_xml(elem, level=level) diff --git a/rally/plugins/common/exporters/junit.py b/rally/plugins/common/exporters/junit.py index b41e87fe07..99b09f672a 100644 --- a/rally/plugins/common/exporters/junit.py +++ b/rally/plugins/common/exporters/junit.py @@ -15,11 +15,8 @@ import datetime as dt import itertools import os -import xml.etree.ElementTree as ET -from rally.common import utils -from rally.common import version -from rally import consts +from rally.common.io import junit from rally.task import exporter @@ -59,57 +56,37 @@ class JUnitXMLExporter(exporter.TaskExporter): """ def generate(self): - root = ET.Element("testsuites") - root.append(ET.Comment("Report is generated by Rally %s at %s" % ( - version.version_string(), - dt.datetime.utcnow().strftime(consts.TimeFormat.ISO8601)))) + root = junit.JUnitXML() for t in self.tasks_results: created_at = dt.datetime.strptime(t["created_at"], "%Y-%m-%dT%H:%M:%S") updated_at = dt.datetime.strptime(t["updated_at"], "%Y-%m-%dT%H:%M:%S") - task = { - "id": t["uuid"], - "tests": 0, - "errors": "0", - "skipped": "0", - "failures": 0, - "time": "%.2f" % (updated_at - created_at).total_seconds(), - "timestamp": t["created_at"], - } - test_cases = [] + test_suite = root.add_test_suite( + id=t["uuid"], + time="%.2f" % (updated_at - created_at).total_seconds(), + timestamp=t["created_at"] + ) for workload in itertools.chain( *[s["workloads"] for s in t["subtasks"]]): class_name, name = workload["name"].split(".", 1) - test_case = { - "id": workload["uuid"], - "time": "%.2f" % workload["full_duration"], - "name": name, - "classname": class_name, - "timestamp": workload["created_at"] - } + test_case = test_suite.add_test_case( + id=workload["uuid"], + time="%.2f" % workload["full_duration"], + classname=class_name, + name=name, + timestamp=workload["created_at"] + ) if not workload["pass_sla"]: - task["failures"] += 1 - test_case["failure"] = "\n".join( + details = "\n".join( [s["detail"] for s in workload["sla_results"]["sla"] - if not s["success"]]) - test_cases.append(test_case) + if not s["success"]] + ) + test_case.mark_as_failed(details) - task["tests"] = str(len(test_cases)) - task["failures"] = str(task["failures"]) - - testsuite = ET.SubElement(root, "testsuite", task) - for test_case in test_cases: - failure = test_case.pop("failure", None) - test_case = ET.SubElement(testsuite, "testcase", test_case) - if failure: - ET.SubElement(test_case, "failure").text = failure - - utils.prettify_xml(root) - - raw_report = ET.tostring(root, encoding="utf-8").decode("utf-8") + raw_report = root.to_string() if self.output_destination: return {"files": {self.output_destination: raw_report}, diff --git a/rally/plugins/common/verification/reporters.py b/rally/plugins/common/verification/reporters.py index b70c45553b..b52f8b5812 100644 --- a/rally/plugins/common/verification/reporters.py +++ b/rally/plugins/common/verification/reporters.py @@ -13,13 +13,10 @@ # under the License. import collections -import datetime as dt import json import re -import xml.etree.ElementTree as ET -from rally.common import utils -from rally.common import version +from rally.common.io import junit from rally import consts from rally.ui import utils as ui_utils from rally.verification import reporter @@ -410,74 +407,55 @@ class JUnitXMLReporter(reporter.VerificationReporter): pass def generate(self): - root = ET.Element("testsuites") - - root.append(ET.Comment("Report is generated by Rally %s at %s" % ( - version.version_string(), - dt.datetime.utcnow().strftime(TIME_FORMAT)))) + report = junit.JUnitXML() for v in self.verifications: - verification = ET.SubElement(root, "testsuite", { - "id": v.uuid, - "time": str(v.tests_duration), - "tests": str(v.tests_count), - "errors": "0", - "skipped": str(v.skipped), - "failures": str(v.failures + v.unexpected_success), - "timestamp": v.created_at.strftime(TIME_FORMAT) - }) + test_suite = report.add_test_suite( + id=v.uuid, + time=str(v.tests_duration), + timestamp=v.created_at.strftime(TIME_FORMAT) + ) + test_suite.setup_final_stats( + tests=str(v.tests_count), + skipped=str(v.skipped), + failures=str(v.failures + v.unexpected_success) + ) + tests = sorted(v.tests.values(), key=lambda t: (t.get("timestamp", ""), t["name"])) for result in tests: class_name, name = result["name"].rsplit(".", 1) - test_case = { - "time": result["duration"], - "name": name, "classname": class_name - } test_id = [tag[3:] for tag in result.get("tags", []) if tag.startswith("id-")] - if test_id: - test_case["id"] = test_id[0] - if "timestamp" in result: - test_case["timestamp"] = result["timestamp"] - test_case_element = ET.SubElement(verification, "testcase", - test_case) + test_case = test_suite.add_test_case( + id=(test_id[0] if test_id else None), + time=result["duration"], name=name, classname=class_name, + timestamp=result.get("timestamp")) + if result["status"] == "success": # nothing to add pass elif result["status"] == "uxsuccess": - # NOTE(andreykurilin): junit doesn't support uxsuccess - # status, so let's display it like "fail" with proper - # comment. - failure = ET.SubElement(test_case_element, "failure") - failure.text = ("It is an unexpected success. The test " - "should fail due to: %s" % - result.get("reason", "Unknown reason")) + test_case.mark_as_uxsuccess( + result.get("reason")) elif result["status"] == "fail": - failure = ET.SubElement(test_case_element, "failure") - failure.text = result.get("traceback", None) + test_case.mark_as_failed( + result.get("traceback", None)) elif result["status"] == "xfail": - # NOTE(andreykurilin): junit doesn't support xfail status, - # so let's display it like "success" with proper comment - test_case_element.append(ET.Comment( - "It is an expected failure due to: %s" % - result.get("reason", "Unknown reason"))) trace = result.get("traceback", None) - if trace: - test_case_element.append(ET.Comment( - "Traceback:\n%s" % trace)) + test_case.mark_as_xfail( + result.get("reason", None), + f"Traceback:\n{trace}" if trace else None) elif result["status"] == "skip": - skipped = ET.SubElement(test_case_element, "skipped") - skipped.text = result.get("reason", "Unknown reason") + test_case.mark_as_skipped( + result.get("reason", None)) else: # wtf is it?! we should add validation of results... pass - utils.prettify_xml(root) - - raw_report = ET.tostring(root, encoding="utf-8").decode("utf-8") + raw_report = report.to_string() if self.output_destination: return {"files": {self.output_destination: raw_report}, "open": self.output_destination} diff --git a/rally/plugins/task/__init__.py b/rally/plugins/task/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/rally/plugins/verification/__init__.py b/rally/plugins/verification/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/ci/playbooks/rally-install/pre.yaml b/tests/ci/playbooks/rally-install/pre.yaml index ec40eb11f0..2709810018 100644 --- a/tests/ci/playbooks/rally-install/pre.yaml +++ b/tests/ci/playbooks/rally-install/pre.yaml @@ -46,6 +46,7 @@ # NOTE(pabelanger): We run apt-get update to ensure we dont have a stale # package cache in the gate. sudo apt-get update + sudo apt-get install python3.6-dev - name: Install bindep shell: diff --git a/tests/unit/common/io/test_junit.py b/tests/unit/common/io/test_junit.py index 71e01e204f..2ab6dcf14f 100644 --- a/tests/unit/common/io/test_junit.py +++ b/tests/unit/common/io/test_junit.py @@ -13,7 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. -import sys +import mock from rally.common.io import junit from tests.unit import test @@ -22,33 +22,40 @@ from tests.unit import test class JUnitTestCase(test.TestCase): def setUp(self): super(JUnitTestCase, self).setUp() - if sys.version_info >= (3, 8): - self.skipTest("This test case is failing due to changed order of " - "xml tag parameters.") + p_mock_datetime = mock.patch("rally.common.io.junit.dt.datetime") + self.mock_datetime = p_mock_datetime.start() + isoformat = self.mock_datetime.utcnow.return_value.isoformat + isoformat.return_value = "TIME" + self.addCleanup(p_mock_datetime.stop) + + p_mock_version = mock.patch("rally.common.version.version_string", + return_value="VERSION") + self.mock_version = p_mock_version.start() + self.addCleanup(p_mock_version.stop) def test_basic_testsuite(self): - j = junit.JUnit("test") - j.add_test("Foo.Bar", 3.14, outcome=junit.JUnit.SUCCESS) - j.add_test("Foo.Baz", 13.37, outcome=junit.JUnit.FAILURE, - message="fail_message") - j.add_test("Eggs.Spam", 42.00, outcome=junit.JUnit.ERROR) + j = junit.JUnitXML() + test_suite = j.add_test_suite("uuid1", time="58.51", timestamp="3") + test_suite.add_test_case(classname="Foo", name="Bar", time="3.14") + t = test_suite.add_test_case(classname="Foo", name="Baz", time="13.37") + t.mark_as_failed("fail_message") - expected = """ - - - - - -""" - self.assertEqual(expected.replace("\n", ""), j.to_xml()) + expected = """ + + + + + fail_message + + + +""" # noqa: E501 + self.assertEqual(expected, j.to_string()) def test_empty_testsuite(self): - j = junit.JUnit("test") - expected = """ -""" - self.assertEqual(expected.replace("\n", ""), j.to_xml()) - - def test_invalid_outcome(self): - j = junit.JUnit("test") - self.assertRaises(ValueError, j.add_test, "Foo.Bar", 1.23, - outcome=1024) + j = junit.JUnitXML() + expected = """ + + +""" + self.assertEqual(expected, str(j)) diff --git a/tests/unit/plugins/common/exporters/test_junit.py b/tests/unit/plugins/common/exporters/test_junit.py index 7fc558d132..642ea9b44a 100644 --- a/tests/unit/plugins/common/exporters/test_junit.py +++ b/tests/unit/plugins/common/exporters/test_junit.py @@ -14,7 +14,6 @@ import datetime as dt import os -import sys import mock @@ -66,20 +65,15 @@ def get_tasks_results(): class JUnitXMLExporterTestCase(test.TestCase): def setUp(self): super(JUnitXMLExporterTestCase, self).setUp() - if sys.version_info >= (3, 8): - self.skipTest("This test case is failing due to changed order of " - "xml tag parameters.") self.datetime = dt.datetime - patcher = mock.patch("rally.plugins.common.exporters.junit.dt") + patcher = mock.patch("rally.common.io.junit.dt") self.dt = patcher.start() - self.dt.datetime.strptime.side_effect = self.datetime.strptime + self.dt.datetime.utcnow.return_value.isoformat.return_value = "$TIME" self.addCleanup(patcher.stop) - @mock.patch("rally.plugins.common.exporters.junit.version.version_string") + @mock.patch("rally.common.version.version_string") def test_generate(self, mock_version_string): - now = self.dt.datetime.utcnow.return_value - now.strftime.return_value = "$TIME" mock_version_string.return_value = "$VERSION" with open(os.path.join(os.path.dirname(__file__), diff --git a/tests/unit/plugins/common/verification/test_reporters.py b/tests/unit/plugins/common/verification/test_reporters.py index 684042c3c0..8152e7823c 100644 --- a/tests/unit/plugins/common/verification/test_reporters.py +++ b/tests/unit/plugins/common/verification/test_reporters.py @@ -15,7 +15,6 @@ import collections import datetime as dt import os -import sys import ddt import mock @@ -388,16 +387,11 @@ class HTMLReporterTestCase(test.TestCase): class JUnitXMLReporterTestCase(test.TestCase): - def setUp(self): - super(JUnitXMLReporterTestCase, self).setUp() - if sys.version_info >= (3, 8): - self.skipTest("This test case is failing due to changed order of " - "xml tag parameters.") - @mock.patch("%s.dt" % PATH) - @mock.patch("%s.version.version_string" % PATH) + @mock.patch("rally.common.io.junit.dt") + @mock.patch("rally.common.version.version_string") def test_generate(self, mock_version_string, mock_dt): - mock_dt.datetime.utcnow.return_value.strftime.return_value = "TIME" + mock_dt.datetime.utcnow.return_value.isoformat.return_value = "TIME" # release when junit reporter was introduced mock_version_string.return_value = "0.8.0" with open(os.path.join(os.path.dirname(__file__),