From a14cb12fad844ce0fe925b2602a1bb47c16b8ff0 Mon Sep 17 00:00:00 2001 From: Rohan Kanade Date: Mon, 23 Jun 2014 15:56:57 +0200 Subject: [PATCH] Adds html result generator for tempest runs * Adds "rally verify results --html --output-file output_to_html_file" flag to generate html result from tempest runs * json2html generates html report similar to subunit2html using raw json from rally database * Use mako templates for html generation Change-Id: Ib390fa02aba4696bacaa4e4ec8b8c2481a54b0d2 Implements: blueprint tempest-subunit-to-json --- rally/cmd/commands/verify.py | 49 +++- .../verifiers/tempest/json2html.py | 135 +++++++++ .../tempest/report_templates/main.mako | 258 ++++++++++++++++++ tests/cmd/commands/test_verify.py | 59 +++- .../verification/verifiers/test_json2html.py | 51 ++++ 5 files changed, 536 insertions(+), 16 deletions(-) create mode 100644 rally/verification/verifiers/tempest/json2html.py create mode 100644 rally/verification/verifiers/tempest/report_templates/main.mako create mode 100644 tests/verification/verifiers/test_json2html.py diff --git a/rally/cmd/commands/verify.py b/rally/cmd/commands/verify.py index 673fdb818c..5dc8e937a5 100644 --- a/rally/cmd/commands/verify.py +++ b/rally/cmd/commands/verify.py @@ -29,6 +29,7 @@ from rally import objects from rally.openstack.common import cliutils as common_cliutils from rally.openstack.common.gettextutils import _ from rally.orchestrator import api +from rally.verification.verifiers.tempest import json2html class VerifyCommands(object): @@ -71,13 +72,26 @@ class VerifyCommands(object): @cliutils.args('--uuid', type=str, dest='verification_uuid', help='UUID of the verification') - @cliutils.args('--pretty', type=str, help=('pretty print (pprint) ' - 'or json print (json)')) - def results(self, verification_uuid, pretty=False): + @cliutils.args('--html', action='store_true', dest='output_html', + help=('Save results in html format to specified file')) + @cliutils.args('--json', action='store_true', dest='output_json', + help=('Save results in json format to specified file')) + @cliutils.args('--pprint', action='store_true', dest='output_pprint', + help=('Save results in pprint format to specified file')) + @cliutils.args('--output-file', type=str, required=False, + dest='output_file', + help='If specified, output will be saved to given file') + def results(self, verification_uuid, output_file=None, output_html=None, + output_json=None, output_pprint=None): """Print raw results of verification. :param verification_uuid: Verification UUID - :param pretty: Pretty print (pprint) or not (json) + :param output_file: If specified, output will be saved to given file + :param output_html: Save results in html format to the specified file + :param output_json: Save results in json format to the specified file + (Default) + :param output_pprint: Save results in pprint format to the + specified file """ try: results = db.verification_result_get(verification_uuid)['data'] @@ -85,14 +99,23 @@ class VerifyCommands(object): print(six.text_type(e)) return 1 - if not pretty or pretty == 'json': - print(json.dumps(results)) - elif pretty == 'pprint': - print() - pprint.pprint(results) - print() + result = '' + if len(filter(lambda x: bool(x), [output_json, output_pprint, + output_html])) > 1: + print("Please specify only on output format") + return 1 + elif output_pprint: + result = pprint.pformat(results) + elif output_html: + result = json2html.main(results) else: - print(_("Wrong value for --pretty=%s") % pretty) + result = json.dumps(results) + + if output_file: + with open(output_file, 'wb') as f: + f.write(result) + else: + print(result) @cliutils.args('--uuid', dest='verification_uuid', type=str, required=False, @@ -119,9 +142,7 @@ class VerifyCommands(object): print ("Total results of verification:\n") total_fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', 'Failures', 'Created at', 'Status'] - common_cliutils.print_list([verification], fields=total_fields, - sortby_index=total_fields.index( - 'Created at')) + common_cliutils.print_list([verification], fields=total_fields) print ("\nTests:\n") fields = ['name', 'time', 'status'] diff --git a/rally/verification/verifiers/tempest/json2html.py b/rally/verification/verifiers/tempest/json2html.py new file mode 100644 index 0000000000..60c559e478 --- /dev/null +++ b/rally/verification/verifiers/tempest/json2html.py @@ -0,0 +1,135 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import mako.template + +__version__ = '0.1' + + +STATUS = {0: 'pass', 1: 'fail', 2: 'error', 3: 'skip'} + +DEFAULT_TITLE = 'Unit Test Report' +DEFAULT_DESCRIPTION = '' + + +class HtmlOutput(object): + """Output test results in html.""" + + def __init__(self, result): + self.success_count = result['success'] + self.failure_count = result['failures'] + self.error_count = result['errors'] + self.skip_count = result['skipped'] + self.total = result['tests'] + self.result = result['test_cases'] + self.abspath = os.path.dirname(__file__) + + def create_report(self): + report_attrs = self._getReportAttributes() + generator = 'json2html %s' % __version__ + heading = self._generate_heading(report_attrs) + report = self._generate_report() + with open("%s/report_templates/main.mako" % self.abspath) as main: + template = mako.template.Template(main.read()) + output = template.render(title=DEFAULT_TITLE, generator=generator, + heading=heading, report=report) + return output.encode('utf8') + + def _getReportAttributes(self): + """Return report attributes as a list of (name, value).""" + status = [] + if self.success_count: + status.append('Pass %s' % self.success_count) + if self.failure_count: + status.append('Failure %s' % self.failure_count) + if self.error_count: + status.append('Error %s' % self.error_count) + if self.skip_count: + status.append('Skip %s' % self.skip_count) + if status: + status = ' '.join(status) + else: + status = 'none' + return [ + ('Status', status), + ] + + def _generate_heading(self, report_attrs): + return dict(title=DEFAULT_TITLE, parameters=report_attrs, + description=DEFAULT_DESCRIPTION) + + def _generate_report(self): + rows = [] + sortedResult = self._sortResult(self.result) + ne = self.error_count + nf = self.failure_count + cid = "c1" + + test_class = dict( + style=(ne > 0 and 'errorClass' or nf > 0 + and 'failClass' or 'passClass'), + desc = "", + count = self.total, + Pass = self.success_count, + fail = nf, + error = ne, + skipped = self.skip_count, + cid = cid + ) + + for tid, name in enumerate(sortedResult): + n = self.result[name]['status'] + o = self.result[name]['output'] + f = self.result[name].get('failure') + e = '' + if f: + e = f['log'] + self._generate_report_test(rows, cid, tid, n, name, o, e) + + return dict(test_class=test_class, tests_list=rows, + count=str(self.success_count + self.failure_count + + self.error_count + self.skip_count), + Pass=str(self.success_count), + fail=str(self.failure_count), + error=str(self.error_count), + skip=str(self.skip_count)) + + def _sortResult(self, results): + # unittest does not seems to run in any particular order. + # Here at least we want to group them together by class. + return sorted(results) + + def _generate_report_test(self, rows, cid, tid, n, name, o, e): + # e.g. 'pt1.1', 'ft1.1', etc + # ptx.x for passed/skipped tests and ftx.x for failed/errored tests. + status_map = {'OK': 0, 'SKIP': 3, 'FAIL': 1, 'ERROR': 2} + n = status_map[n] + tid = ((n == 0 or n == 3) and + 'p' or 'f') + 't%s.%s' % (cid, tid + 1) + desc = name + + row = dict( + tid=tid, + Class=((n == 0 or n == 3) and 'hiddenRow' or 'none'), + style=(n == 2 and 'errorCase' or + (n == 1 and 'failCase' or 'none')), + desc=desc, + output=o + e, + status=STATUS[n], + ) + rows.append(row) + + +def main(result): + return HtmlOutput(result).create_report() diff --git a/rally/verification/verifiers/tempest/report_templates/main.mako b/rally/verification/verifiers/tempest/report_templates/main.mako new file mode 100644 index 0000000000..eb3bd88c72 --- /dev/null +++ b/rally/verification/verifiers/tempest/report_templates/main.mako @@ -0,0 +1,258 @@ + + + + + + ${title} + + + + + + +
+

${heading['title']}

+ % for name, value in heading['parameters']: +

${name}: ${value}

+ % endfor +

${heading['description']}

+
+ +

Show + Summary + Failed + All +

+ + + + + + + + + + + + + + + + + + + + + + <% + test_class = report['test_class'] + tests_list = report['tests_list'] + cid = test_class['cid'] + count = test_class['count'] + %> + + + + + + + + + + + + + % for test in tests_list: + + % if 'output' in test: + + + + + + + % else: + + + + + % endif + + + % endfor + + + + + + + + + + + +
Test Group/Test caseCountPassFailErrorSkipView
${test_class['desc']}${test_class['count']}${test_class['Pass']}${test_class['fail']}${test_class['error']}${test_class['skipped']}Detail
${test['desc']}
 + + + + ${test['status']} + + + + +
${test['desc']}
${test['status']}
Total${report['count']}${report['Pass']}${report['fail']}${report['error']}${report['skip']}  
+ + +
 
+ + + + diff --git a/tests/cmd/commands/test_verify.py b/tests/cmd/commands/test_verify.py index 56e05ff00b..6931d075f1 100644 --- a/tests/cmd/commands/test_verify.py +++ b/tests/cmd/commands/test_verify.py @@ -13,11 +13,14 @@ # License for the specific language governing permissions and limitations # under the License. +import os + import mock import six from rally.cmd.commands import verify from rally import consts +from rally import exceptions from rally import objects from tests import test @@ -114,8 +117,60 @@ class VerifyCommandsTestCase(test.TestCase): six.itervalues(tests.data['test_cases'])) self.verify.show(verification_id) mock_print_list.assert_any_call( - [verification], fields=total_fields, - sortby_index=total_fields.index('Created at')) + [verification], fields=total_fields) mock_verification_get.assert_called_once_with(verification_id) mock_verification_result_get.assert_called_once_with(verification_id) mock_print_list.assert_any_call(values, fields, sortby_index=0) + + @mock.patch('rally.db.verification_result_get', return_value={'data': {}}) + @mock.patch('json.dumps') + def test_results(self, mock_json_dumps, mock_db_result_get): + verification_uuid = 'a0231bdf-6a4e-4daf-8ab1-ae076f75f070' + self.verify.results(verification_uuid, output_json=True) + + mock_db_result_get.assert_called_once_with(verification_uuid) + mock_json_dumps.assert_called_once_with({}) + + @mock.patch('rally.db.verification_result_get') + def test_results_verification_not_found(self, mock_db_result_get): + verification_uuid = '9044ced5-9c84-4666-8a8f-4b73a2b62acb' + mock_db_result_get.side_effect = exceptions.NotFoundException() + self.assertEqual(self.verify.results(verification_uuid), 1) + + mock_db_result_get.assert_called_once_with(verification_uuid) + + @mock.patch('rally.db.verification_result_get', return_value={'data': {}}) + def test_results_with_output_json_and_output_file(self, + mock_db_result_get): + verification_uuid = '94615cd4-ff45-4123-86bd-4b0741541d09' + self.verify.results(verification_uuid, output_file='results', + output_json=True) + + mock_db_result_get.assert_called_once_with(verification_uuid) + self.assertTrue(os.path.isfile('results')) + + @mock.patch('rally.db.verification_result_get', return_value={'data': {}}) + def test_results_with_output_pprint_and_output_file(self, + mock_db_result_get): + verification_uuid = 'fa882ccc-153e-4a6e-9001-91fecda6a75c' + self.verify.results(verification_uuid, output_pprint=True, + output_file='results') + + mock_db_result_get.assert_called_once_with(verification_uuid) + self.assertTrue(os.path.isfile('results')) + + @mock.patch('rally.db.verification_result_get') + @mock.patch('rally.verification.verifiers.tempest.json2html.main', + return_value='') + def test_results_with_output_html_and_output_file(self, + mock_json2html_main, + mock_db_result_get): + verification_uuid = '7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa' + results = {'data': {}} + mock_db_result_get.return_value = results + self.verify.results(verification_uuid, output_html=True, + output_file='results') + + mock_db_result_get.assert_called_once_with(verification_uuid) + mock_json2html_main.assert_called_once() + self.assertTrue(os.path.isfile('results')) diff --git a/tests/verification/verifiers/test_json2html.py b/tests/verification/verifiers/test_json2html.py new file mode 100644 index 0000000000..035ffb35d9 --- /dev/null +++ b/tests/verification/verifiers/test_json2html.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from rally.verification.verifiers.tempest import json2html +from tests import test + + +class Json2HtmlTestCase(test.TestCase): + + def test_main(self): + + data = {'tests': 4, 'skipped': 1, 'errors': 1, 'failures': 1, + 'success': 1, 'time': 22, + 'test_cases': { + 'tp': {'name': 'tp', 'time': 2, 'status': 'OK', + 'output': 'tp_ok'}, + 'ts': {'name': 'ts', 'time': 4, 'status': 'SKIP', + 'output': 'ts_skip'}, + 'tf': {'name': 'tf', 'time': 6, 'status': 'FAIL', + 'output': 'tf_fail', + 'failure': {'type': 'tf', 'log': 'fail_log'}}, + 'te': {'name': 'te', 'time': 2, 'status': 'ERROR', + 'output': 'te_error', + 'failure': {'type': 'te', 'log': 'error+log'}}}} + + obj = json2html.HtmlOutput(data) + self.assertEqual(obj.success_count, data['success']) + self.assertEqual(obj.failure_count, data['failures']) + self.assertEqual(obj.skip_count, data['skipped']) + self.assertEqual(obj.error_count, data['errors']) + + report_attrs = obj._getReportAttributes() + generator = 'json2html %s' % json2html.__version__ + heading = obj._generate_heading(report_attrs) + report = obj._generate_report() + with mock.patch('mako.template.Template') as mock_mako: + obj.create_report() + mock_mako().render.assert_called_once_with( + title=json2html.DEFAULT_TITLE, generator=generator, + heading=heading, report=report)