openstack/rally
Code Issues Proposed changes

Adds html result generator for tempest runs

Browse Source 
* Adds "rally verify results --html --output-file output_to_html_file" flag to generate html result from
  tempest runs
* json2html generates html report similar to subunit2html using raw json
  from rally database
* Use mako templates for html generation

Change-Id: Ib390fa02aba4696bacaa4e4ec8b8c2481a54b0d2
Implements: blueprint tempest-subunit-to-json
This commit is contained in:
Rohan Kanade 2014-06-23 15:56:57 +02:00
parent d1e007e402
commit a14cb12fad
5 changed files with 536 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
rally/cmd/commands/verify.py
View File

@ -29,6 +29,7 @@ from rally import objects
from rally.openstack.common import cliutils as common_cliutils
from rally.openstack.common.gettextutils import _
from rally.orchestrator import api
from rally.verification.verifiers.tempest import json2html
class VerifyCommands(object):
@ -71,13 +72,26 @@ class VerifyCommands(object):
@cliutils.args('--uuid', type=str, dest='verification_uuid',
help='UUID of the verification')
@cliutils.args('--pretty', type=str, help=('pretty print (pprint) '
'or json print (json)'))
def results(self, verification_uuid, pretty=False):
@cliutils.args('--html', action='store_true', dest='output_html',
help=('Save results in html format to specified file'))
@cliutils.args('--json', action='store_true', dest='output_json',
help=('Save results in json format to specified file'))
@cliutils.args('--pprint', action='store_true', dest='output_pprint',
help=('Save results in pprint format to specified file'))
@cliutils.args('--output-file', type=str, required=False,
dest='output_file',
help='If specified, output will be saved to given file')
def results(self, verification_uuid, output_file=None, output_html=None,
output_json=None, output_pprint=None):
"""Print raw results of verification.
:param verification_uuid: Verification UUID
:param pretty: Pretty print (pprint) or not (json)
:param output_file: If specified, output will be saved to given file
:param output_html: Save results in html format to the specified file
:param output_json: Save results in json format to the specified file
(Default)
:param output_pprint: Save results in pprint format to the
specified file
"""
try:
results = db.verification_result_get(verification_uuid)['data']
@ -85,14 +99,23 @@ class VerifyCommands(object):
print(six.text_type(e))
return 1
if not pretty or pretty == 'json':
print(json.dumps(results))
elif pretty == 'pprint':
print()
pprint.pprint(results)
print()
result = ''
if len(filter(lambda x: bool(x), [output_json, output_pprint,
output_html])) > 1:
print("Please specify only on output format")
return 1
elif output_pprint:
result = pprint.pformat(results)
elif output_html:
result = json2html.main(results)
else:
print(_("Wrong value for --pretty=%s") % pretty)
result = json.dumps(results)
if output_file:
with open(output_file, 'wb') as f:
f.write(result)
else:
print(result)
@cliutils.args('--uuid', dest='verification_uuid', type=str,
required=False,
@ -119,9 +142,7 @@ class VerifyCommands(object):
print ("Total results of verification:\n")
total_fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests',
'Failures', 'Created at', 'Status']
common_cliutils.print_list([verification], fields=total_fields,
sortby_index=total_fields.index(
'Created at'))
common_cliutils.print_list([verification], fields=total_fields)
print ("\nTests:\n")
fields = ['name', 'time', 'status']

135
rally/verification/verifiers/tempest/json2html.py Normal file
View File

@ -0,0 +1,135 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import mako.template
__version__ = '0.1'
STATUS = {0: 'pass', 1: 'fail', 2: 'error', 3: 'skip'}
DEFAULT_TITLE = 'Unit Test Report'
DEFAULT_DESCRIPTION = ''
class HtmlOutput(object):
"""Output test results in html."""
def __init__(self, result):
self.success_count = result['success']
self.failure_count = result['failures']
self.error_count = result['errors']
self.skip_count = result['skipped']
self.total = result['tests']
self.result = result['test_cases']
self.abspath = os.path.dirname(__file__)
def create_report(self):
report_attrs = self._getReportAttributes()
generator = 'json2html %s' % __version__
heading = self._generate_heading(report_attrs)
report = self._generate_report()
with open("%s/report_templates/main.mako" % self.abspath) as main:
template = mako.template.Template(main.read())
output = template.render(title=DEFAULT_TITLE, generator=generator,
heading=heading, report=report)
return output.encode('utf8')
def _getReportAttributes(self):
"""Return report attributes as a list of (name, value)."""
status = []
if self.success_count:
status.append('Pass %s' % self.success_count)
if self.failure_count:
status.append('Failure %s' % self.failure_count)
if self.error_count:
status.append('Error %s' % self.error_count)
if self.skip_count:
status.append('Skip %s' % self.skip_count)
if status:
status = ' '.join(status)
else:
status = 'none'
return [
('Status', status),
]
def _generate_heading(self, report_attrs):
return dict(title=DEFAULT_TITLE, parameters=report_attrs,
description=DEFAULT_DESCRIPTION)
def _generate_report(self):
rows = []
sortedResult = self._sortResult(self.result)
ne = self.error_count
nf = self.failure_count
cid = "c1"
test_class = dict(
style=(ne > 0 and 'errorClass' or nf > 0
and 'failClass' or 'passClass'),
desc = "",
count = self.total,
Pass = self.success_count,
fail = nf,
error = ne,
skipped = self.skip_count,
cid = cid
)
for tid, name in enumerate(sortedResult):
n = self.result[name]['status']
o = self.result[name]['output']
f = self.result[name].get('failure')
e = ''
if f:
e = f['log']
self._generate_report_test(rows, cid, tid, n, name, o, e)
return dict(test_class=test_class, tests_list=rows,
count=str(self.success_count + self.failure_count +
self.error_count + self.skip_count),
Pass=str(self.success_count),
fail=str(self.failure_count),
error=str(self.error_count),
skip=str(self.skip_count))
def _sortResult(self, results):
# unittest does not seems to run in any particular order.
# Here at least we want to group them together by class.
return sorted(results)
def _generate_report_test(self, rows, cid, tid, n, name, o, e):
# e.g. 'pt1.1', 'ft1.1', etc
# ptx.x for passed/skipped tests and ftx.x for failed/errored tests.
status_map = {'OK': 0, 'SKIP': 3, 'FAIL': 1, 'ERROR': 2}
n = status_map[n]
tid = ((n == 0 or n == 3) and
'p' or 'f') + 't%s.%s' % (cid, tid + 1)
desc = name
row = dict(
tid=tid,
Class=((n == 0 or n == 3) and 'hiddenRow' or 'none'),
style=(n == 2 and 'errorCase' or
(n == 1 and 'failCase' or 'none')),
desc=desc,
output=o + e,
status=STATUS[n],
)
rows.append(row)
def main(result):
return HtmlOutput(result).create_report()

258
rally/verification/verifiers/tempest/report_templates/main.mako Normal file
View File

@ -0,0 +1,258 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>${title}</title>
<meta name="generator" content="${generator}"/>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
<style type="text/css" media="screen">
body {
font-family: verdana, arial, helvetica, sans-serif;
font-size: 80%;
}
table {
font-size: 100%; width: 100%;
}
pre {
font-size: 80%;
}
h1 {
font-size: 16pt;
color: gray;
}
.heading {
margin-top: 0ex;
margin-bottom: 1ex;
}
.heading .attribute {
margin-top: 1ex;
margin-bottom: 0;
}
.heading .description {
margin-top: 4ex;
margin-bottom: 6ex;
}
a.popup_link {
}
a.popup_link:hover {
color: red;
}
.popup_window {
display: none;
overflow-x: scroll;
padding: 10px;
background-color: #E6E6D6;
font-family: "Ubuntu Mono", "Lucida Console", "Courier New", monospace;
text-align: left;
font-size: 8pt;
}
}
#show_detail_line {
margin-top: 3ex;
margin-bottom: 1ex;
}
#result_table {
width: 100%;
border-collapse: collapse;
border: 1px solid #777;
}
#header_row {
font-weight: bold;
color: white;
background-color: #777;
}
#result_table td {
border: 1px solid #777;
padding: 2px;
}
#total_row { font-weight: bold; }
.passClass { background-color: #6c6; }
.failClass { background-color: #c60; }
.errorClass { background-color: #c00; }
.passCase { color: #6c6; }
.failCase { color: #c60; font-weight: bold; }
.errorCase { color: #c00; font-weight: bold; }
.hiddenRow { display: none; }
.testcase { margin-left: 2em; }
td.testname {width: 40%}
td.small {width: 40px}
</style>
</head>
<body>
<div class='heading'>
<h1>${heading['title']}</h1>
% for name, value in heading['parameters']:
<p class='attribute'><strong>${name}:</strong> ${value}</p>
% endfor
<p class='description'>${heading['description']}</p>
</div>
<p id='show_detail_line'>Show
<a href='#' onclick='showCase(0);return false;'>Summary</a>
<a href='#' onclick='showCase(1);return false;'>Failed</a>
<a href='#' onclick='showCase(2);return false;'>All</a>
</p>
<table id='result_table'>
<colgroup>
<col align='left' />
<col align='right' />
<col align='right' />
<col align='right' />
<col align='right' />
<col align='right' />
<col align='right' />
<col align='right' />
</colgroup>
<tr id='header_row'>
<td>Test Group/Test case</td>
<td>Count</td>
<td>Pass</td>
<td>Fail</td>
<td>Error</td>
<td>Skip</td>
<td>View</td>
<td> </td>
</tr>
<%
test_class = report['test_class']
tests_list = report['tests_list']
cid = test_class['cid']
count = test_class['count']
%>
<tr class="${test_class['style']}">
<td class="testname">${test_class['desc']}</td>
<td class="small">${test_class['count']}</td>
<td class="small">${test_class['Pass']}</td>
<td class="small">${test_class['fail']}</td>
<td class="small">${test_class['error']}</td>
<td class="small">${test_class['skipped']}</td>
<td class="small"><a href='#' onclick="showClassDetail('${cid}',${count});return false;">Detail</a></td>
<td> </td>
</tr>
% for test in tests_list:
% if 'output' in test:
<tr id="${test['tid']}" class="${test['Class']}">
<td class="${test['style']}"><div class='testcase'>${test['desc']}</div></td>
<td colspan='7' align='left'>
<!--css div popup start-->
<a class="popup_link" onfocus='this.blur();'
href='javascript:showTestDetail("div_${test['tid']}")' >
${test['status']}</a>
<div id="div_${test['tid']}" class="popup_window">
<div style='text-align: right; color:red;cursor:pointer'>
<a onfocus='this.blur();'
onclick="document.getElementById('div_${test['tid']}').style.display = 'none' ">
[x]</a>
</div>
<pre>
${test['tid']}: ${test['output']}
</pre>
</div>
<!--css div popup end-->
</td>
</tr>
% else:
<tr id="${test['tid']}" class="${test['Class']}">
<td class="${test['style']}"><div class='testcase'>${test['desc']}</div></td>
<td colspan='6' align='center'>${test['status']}</td>
</tr>
% endif
% endfor
<tr id='total_row'>
<td>Total</td>
<td>${report['count']}</td>
<td>${report['Pass']}</td>
<td>${report['fail']}</td>
<td>${report['error']}</td>
<td>${report['skip']}</td>
<td>&nbsp;</td>
<td>&nbsp;</td>
</tr>
</table>
<div>&nbsp;</div>
<script language="javascript" type="text/javascript">
/* level - 0:Summary; 1:Failed; 2:All */
function showCase(level) {
var trs = document.getElementsByTagName("tr");
for (var i = 0; i < trs.length; i++){
switch(trs[i].id.substr(0, 2)) {
case "ft":
trs[i].className = (level < 1) ? "hiddenRow" : "";
break
case "pt":
trs[i].className = (level > 1) ? "" : "hiddenRow";
break
}
}
}
function getById(id){
return document.getElementById(id);
}
function showClassDetail(cid, count) {
var id_list = Array(count);
var toHide = 1;
for (var i = 0; i < count; i++) {
var tid0 = 't' + cid.substr(1) + '.' + (i+1);
var tr = getById(tid);
var tid = (tr) ? 'f' + tid0 : 'p' + tid0;
id_list[i] = tid;
if (tr.className) {
toHide = 0;
}
}
for (var i = 0; i < count; i++) {
tid = id_list[i];
if (toHide) {
getById('div_'+tid).style.display = 'none'
getById(tid).className = 'hiddenRow';
}
else {
getById(tid).className = '';
}
}
}
function showTestDetail(div_id){
var div = getById(div_id);
div.style.display = (div.style.display != "block") ? "block" : "none";
}
function html_escape(s) {
return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
}
</script>
</body>
</html>

59
tests/cmd/commands/test_verify.py
View File

@ -13,11 +13,14 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
import mock
import six
from rally.cmd.commands import verify
from rally import consts
from rally import exceptions
from rally import objects
from tests import test
@ -114,8 +117,60 @@ class VerifyCommandsTestCase(test.TestCase):
six.itervalues(tests.data['test_cases']))
self.verify.show(verification_id)
mock_print_list.assert_any_call(
[verification], fields=total_fields,
sortby_index=total_fields.index('Created at'))
[verification], fields=total_fields)
mock_verification_get.assert_called_once_with(verification_id)
mock_verification_result_get.assert_called_once_with(verification_id)
mock_print_list.assert_any_call(values, fields, sortby_index=0)
@mock.patch('rally.db.verification_result_get', return_value={'data': {}})
@mock.patch('json.dumps')
def test_results(self, mock_json_dumps, mock_db_result_get):
verification_uuid = 'a0231bdf-6a4e-4daf-8ab1-ae076f75f070'
self.verify.results(verification_uuid, output_json=True)
mock_db_result_get.assert_called_once_with(verification_uuid)
mock_json_dumps.assert_called_once_with({})
@mock.patch('rally.db.verification_result_get')
def test_results_verification_not_found(self, mock_db_result_get):
verification_uuid = '9044ced5-9c84-4666-8a8f-4b73a2b62acb'
mock_db_result_get.side_effect = exceptions.NotFoundException()
self.assertEqual(self.verify.results(verification_uuid), 1)
mock_db_result_get.assert_called_once_with(verification_uuid)
@mock.patch('rally.db.verification_result_get', return_value={'data': {}})
def test_results_with_output_json_and_output_file(self,
mock_db_result_get):
verification_uuid = '94615cd4-ff45-4123-86bd-4b0741541d09'
self.verify.results(verification_uuid, output_file='results',
output_json=True)
mock_db_result_get.assert_called_once_with(verification_uuid)
self.assertTrue(os.path.isfile('results'))
@mock.patch('rally.db.verification_result_get', return_value={'data': {}})
def test_results_with_output_pprint_and_output_file(self,
mock_db_result_get):
verification_uuid = 'fa882ccc-153e-4a6e-9001-91fecda6a75c'
self.verify.results(verification_uuid, output_pprint=True,
output_file='results')
mock_db_result_get.assert_called_once_with(verification_uuid)
self.assertTrue(os.path.isfile('results'))
@mock.patch('rally.db.verification_result_get')
@mock.patch('rally.verification.verifiers.tempest.json2html.main',
return_value='')
def test_results_with_output_html_and_output_file(self,
mock_json2html_main,
mock_db_result_get):
verification_uuid = '7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa'
results = {'data': {}}
mock_db_result_get.return_value = results
self.verify.results(verification_uuid, output_html=True,
output_file='results')
mock_db_result_get.assert_called_once_with(verification_uuid)
mock_json2html_main.assert_called_once()
self.assertTrue(os.path.isfile('results'))

51
tests/verification/verifiers/test_json2html.py Normal file
View File

@ -0,0 +1,51 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from rally.verification.verifiers.tempest import json2html
from tests import test
class Json2HtmlTestCase(test.TestCase):
def test_main(self):
data = {'tests': 4, 'skipped': 1, 'errors': 1, 'failures': 1,
'success': 1, 'time': 22,
'test_cases': {
'tp': {'name': 'tp', 'time': 2, 'status': 'OK',
'output': 'tp_ok'},
'ts': {'name': 'ts', 'time': 4, 'status': 'SKIP',
'output': 'ts_skip'},
'tf': {'name': 'tf', 'time': 6, 'status': 'FAIL',
'output': 'tf_fail',
'failure': {'type': 'tf', 'log': 'fail_log'}},
'te': {'name': 'te', 'time': 2, 'status': 'ERROR',
'output': 'te_error',
'failure': {'type': 'te', 'log': 'error+log'}}}}
obj = json2html.HtmlOutput(data)
self.assertEqual(obj.success_count, data['success'])
self.assertEqual(obj.failure_count, data['failures'])
self.assertEqual(obj.skip_count, data['skipped'])
self.assertEqual(obj.error_count, data['errors'])
report_attrs = obj._getReportAttributes()
generator = 'json2html %s' % json2html.__version__
heading = obj._generate_heading(report_attrs)
report = obj._generate_report()
with mock.patch('mako.template.Template') as mock_mako:
obj.create_report()
mock_mako().render.assert_called_once_with(
title=json2html.DEFAULT_TITLE, generator=generator,
heading=heading, report=report)