Adds html result generator for tempest runs

* Adds "rally verify results --html --output-file output_to_html_file" flag to generate html result from
  tempest runs
* json2html generates html report similar to subunit2html using raw json
  from rally database
* Use mako templates for html generation

Change-Id: Ib390fa02aba4696bacaa4e4ec8b8c2481a54b0d2
Implements: blueprint tempest-subunit-to-json

rally/cmd/commands/verify.py

@@ -29,6 +29,7 @@ from rally import objects
 from rally.openstack.common import cliutils as common_cliutils
 from rally.openstack.common.gettextutils import _
 from rally.orchestrator import api
+from rally.verification.verifiers.tempest import json2html
 
 
 class VerifyCommands(object):
@@ -71,13 +72,26 @@ class VerifyCommands(object):
 
     @cliutils.args('--uuid', type=str, dest='verification_uuid',
                    help='UUID of the verification')
-    @cliutils.args('--pretty', type=str, help=('pretty print (pprint) '
+    @cliutils.args('--html', action='store_true', dest='output_html',
-                                               'or json print (json)'))
+                   help=('Save results in html format to specified file'))
-    def results(self, verification_uuid, pretty=False):
+    @cliutils.args('--json', action='store_true', dest='output_json',
+                   help=('Save results in json format to specified file'))
+    @cliutils.args('--pprint', action='store_true', dest='output_pprint',
+                   help=('Save results in pprint format to specified file'))
+    @cliutils.args('--output-file', type=str, required=False,
+                   dest='output_file',
+                   help='If specified, output will be saved to given file')
+    def results(self, verification_uuid, output_file=None, output_html=None,
+                output_json=None, output_pprint=None):
         """Print raw results of verification.
 
         :param verification_uuid: Verification UUID
-        :param pretty: Pretty print (pprint) or not (json)
+        :param output_file: If specified, output will be saved to given file
+        :param output_html: Save results in html format to the specified file
+        :param output_json: Save results in json format to the specified file
+                            (Default)
+        :param output_pprint: Save results in pprint format to the
+                              specified file
         """
         try:
             results = db.verification_result_get(verification_uuid)['data']
@@ -85,14 +99,23 @@ class VerifyCommands(object):
             print(six.text_type(e))
             return 1
 
-        if not pretty or pretty == 'json':
+        result = ''
-            print(json.dumps(results))
+        if len(filter(lambda x: bool(x), [output_json, output_pprint,
-        elif pretty == 'pprint':
+                                          output_html])) > 1:
-            print()
+            print("Please specify only on output format")
-            pprint.pprint(results)
+            return 1
-            print()
+        elif output_pprint:
+            result = pprint.pformat(results)
+        elif output_html:
+            result = json2html.main(results)
         else:
-            print(_("Wrong value for --pretty=%s") % pretty)
+            result = json.dumps(results)
+
+        if output_file:
+            with open(output_file, 'wb') as f:
+                f.write(result)
+        else:
+            print(result)
 
     @cliutils.args('--uuid', dest='verification_uuid', type=str,
                    required=False,
@@ -119,9 +142,7 @@ class VerifyCommands(object):
         print ("Total results of verification:\n")
         total_fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests',
                         'Failures', 'Created at', 'Status']
-        common_cliutils.print_list([verification], fields=total_fields,
+        common_cliutils.print_list([verification], fields=total_fields)
-                                   sortby_index=total_fields.index(
-                                       'Created at'))
 
         print ("\nTests:\n")
         fields = ['name', 'time', 'status']

rally/verification/verifiers/tempest/json2html.py

@@ -0,0 +1,135 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import mako.template
+
+__version__ = '0.1'
+
+
+STATUS = {0: 'pass', 1: 'fail', 2: 'error', 3: 'skip'}
+
+DEFAULT_TITLE = 'Unit Test Report'
+DEFAULT_DESCRIPTION = ''
+
+
+class HtmlOutput(object):
+    """Output test results in html."""
+
+    def __init__(self, result):
+        self.success_count = result['success']
+        self.failure_count = result['failures']
+        self.error_count = result['errors']
+        self.skip_count = result['skipped']
+        self.total = result['tests']
+        self.result = result['test_cases']
+        self.abspath = os.path.dirname(__file__)
+
+    def create_report(self):
+        report_attrs = self._getReportAttributes()
+        generator = 'json2html %s' % __version__
+        heading = self._generate_heading(report_attrs)
+        report = self._generate_report()
+        with open("%s/report_templates/main.mako" % self.abspath) as main:
+            template = mako.template.Template(main.read())
+            output = template.render(title=DEFAULT_TITLE, generator=generator,
+                                     heading=heading, report=report)
+        return output.encode('utf8')
+
+    def _getReportAttributes(self):
+        """Return report attributes as a list of (name, value)."""
+        status = []
+        if self.success_count:
+            status.append('Pass %s' % self.success_count)
+        if self.failure_count:
+            status.append('Failure %s' % self.failure_count)
+        if self.error_count:
+            status.append('Error %s' % self.error_count)
+        if self.skip_count:
+            status.append('Skip %s' % self.skip_count)
+        if status:
+            status = ' '.join(status)
+        else:
+            status = 'none'
+        return [
+            ('Status', status),
+        ]
+
+    def _generate_heading(self, report_attrs):
+        return dict(title=DEFAULT_TITLE, parameters=report_attrs,
+                    description=DEFAULT_DESCRIPTION)
+
+    def _generate_report(self):
+        rows = []
+        sortedResult = self._sortResult(self.result)
+        ne = self.error_count
+        nf = self.failure_count
+        cid = "c1"
+
+        test_class = dict(
+            style=(ne > 0 and 'errorClass' or nf > 0
+                   and 'failClass' or 'passClass'),
+            desc = "",
+            count = self.total,
+            Pass = self.success_count,
+            fail = nf,
+            error = ne,
+            skipped = self.skip_count,
+            cid = cid
+        )
+
+        for tid, name in enumerate(sortedResult):
+            n = self.result[name]['status']
+            o = self.result[name]['output']
+            f = self.result[name].get('failure')
+            e = ''
+            if f:
+                e = f['log']
+            self._generate_report_test(rows, cid, tid, n, name, o, e)
+
+        return dict(test_class=test_class, tests_list=rows,
+                    count=str(self.success_count + self.failure_count +
+                              self.error_count + self.skip_count),
+                    Pass=str(self.success_count),
+                    fail=str(self.failure_count),
+                    error=str(self.error_count),
+                    skip=str(self.skip_count))
+
+    def _sortResult(self, results):
+        # unittest does not seems to run in any particular order.
+        # Here at least we want to group them together by class.
+        return sorted(results)
+
+    def _generate_report_test(self, rows, cid, tid, n, name, o, e):
+        # e.g. 'pt1.1', 'ft1.1', etc
+        # ptx.x for passed/skipped tests and ftx.x for failed/errored tests.
+        status_map = {'OK': 0, 'SKIP': 3, 'FAIL': 1, 'ERROR': 2}
+        n = status_map[n]
+        tid = ((n == 0 or n == 3) and
+               'p' or 'f') + 't%s.%s' % (cid, tid + 1)
+        desc = name
+
+        row = dict(
+            tid=tid,
+            Class=((n == 0 or n == 3) and 'hiddenRow' or 'none'),
+            style=(n == 2 and 'errorCase' or
+                   (n == 1 and 'failCase' or 'none')),
+            desc=desc,
+            output=o + e,
+            status=STATUS[n],
+        )
+        rows.append(row)
+
+
+def main(result):
+    return HtmlOutput(result).create_report()

rally/verification/verifiers/tempest/report_templates/main.mako

@@ -0,0 +1,258 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+    <head>
+        <title>${title}</title>
+        <meta name="generator" content="${generator}"/>
+        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+        <style type="text/css" media="screen">
+            body {
+                    font-family: verdana, arial, helvetica, sans-serif;
+                    font-size: 80%;
+            }
+
+            table {
+                    font-size: 100%; width: 100%;
+            }
+
+            pre {
+                    font-size: 80%;
+            }
+
+            h1 {
+                    font-size: 16pt;
+                    color: gray;
+            }
+            .heading {
+                margin-top: 0ex;
+                margin-bottom: 1ex;
+            }
+
+            .heading .attribute {
+                margin-top: 1ex;
+                margin-bottom: 0;
+            }
+
+            .heading .description {
+                margin-top: 4ex;
+                margin-bottom: 6ex;
+            }
+
+            a.popup_link {
+            }
+
+            a.popup_link:hover {
+                color: red;
+            }
+
+            .popup_window {
+                display: none;
+                overflow-x: scroll;
+                padding: 10px;
+                background-color: #E6E6D6;
+                font-family: "Ubuntu Mono", "Lucida Console", "Courier New", monospace;
+                text-align: left;
+                font-size: 8pt;
+            }
+
+            }
+
+            #show_detail_line {
+                margin-top: 3ex;
+                margin-bottom: 1ex;
+            }
+            #result_table {
+                width: 100%;
+                border-collapse: collapse;
+                border: 1px solid #777;
+            }
+            #header_row {
+                font-weight: bold;
+                color: white;
+                background-color: #777;
+            }
+            #result_table td {
+                border: 1px solid #777;
+                padding: 2px;
+            }
+            #total_row  { font-weight: bold; }
+            .passClass  { background-color: #6c6; }
+            .failClass  { background-color: #c60; }
+            .errorClass { background-color: #c00; }
+            .passCase   { color: #6c6; }
+            .failCase   { color: #c60; font-weight: bold; }
+            .errorCase  { color: #c00; font-weight: bold; }
+            .hiddenRow  { display: none; }
+            .testcase   { margin-left: 2em; }
+            td.testname {width: 40%}
+            td.small {width: 40px}
+
+
+        </style>
+    </head>
+
+    <body>
+        <div class='heading'>
+            <h1>${heading['title']}</h1>
+            % for name, value in heading['parameters']:
+                <p class='attribute'><strong>${name}:</strong> ${value}</p>
+                % endfor
+                <p class='description'>${heading['description']}</p>
+        </div>
+
+        <p id='show_detail_line'>Show
+        <a href='#' onclick='showCase(0);return false;'>Summary</a>
+        <a href='#' onclick='showCase(1);return false;'>Failed</a>
+        <a href='#' onclick='showCase(2);return false;'>All</a>
+        </p>
+        <table id='result_table'>
+            <colgroup>
+            <col align='left' />
+            <col align='right' />
+            <col align='right' />
+            <col align='right' />
+            <col align='right' />
+            <col align='right' />
+            <col align='right' />
+            <col align='right' />
+            </colgroup>
+            <tr id='header_row'>
+                <td>Test Group/Test case</td>
+                <td>Count</td>
+                <td>Pass</td>
+                <td>Fail</td>
+                <td>Error</td>
+                <td>Skip</td>
+                <td>View</td>
+                <td> </td>
+            </tr>
+            <%
+            test_class = report['test_class']
+            tests_list = report['tests_list']
+            cid = test_class['cid']
+            count = test_class['count']
+            %>
+            <tr class="${test_class['style']}">
+                <td class="testname">${test_class['desc']}</td>
+                <td class="small">${test_class['count']}</td>
+                <td class="small">${test_class['Pass']}</td>
+                <td class="small">${test_class['fail']}</td>
+                <td class="small">${test_class['error']}</td>
+                <td class="small">${test_class['skipped']}</td>
+
+                <td class="small"><a href='#' onclick="showClassDetail('${cid}',${count});return false;">Detail</a></td>
+                <td> </td>
+            </tr>
+
+            % for test in tests_list:
+
+            % if 'output' in test:
+              <tr id="${test['tid']}" class="${test['Class']}">
+                <td class="${test['style']}"><div class='testcase'>${test['desc']}</div></td>
+                <td colspan='7' align='left'>
+
+                <!--css div popup start-->
+                <a class="popup_link" onfocus='this.blur();'
+                href='javascript:showTestDetail("div_${test['tid']}")' >
+                    ${test['status']}</a>
+
+                <div id="div_${test['tid']}" class="popup_window">
+                    <div style='text-align: right; color:red;cursor:pointer'>
+                    <a onfocus='this.blur();'
+                    onclick="document.getElementById('div_${test['tid']}').style.display = 'none' ">
+                       [x]</a>
+                    </div>
+                    <pre>
+                    ${test['tid']}: ${test['output']}
+                    </pre>
+                </div>
+                <!--css div popup end-->
+
+                </td>
+            </tr>
+
+
+            % else:
+            <tr id="${test['tid']}" class="${test['Class']}">
+                <td class="${test['style']}"><div class='testcase'>${test['desc']}</div></td>
+                <td colspan='6' align='center'>${test['status']}</td>
+            </tr>
+            % endif
+
+
+            % endfor
+
+            <tr id='total_row'>
+                <td>Total</td>
+                <td>${report['count']}</td>
+                <td>${report['Pass']}</td>
+                <td>${report['fail']}</td>
+                <td>${report['error']}</td>
+                <td>${report['skip']}</td>
+                <td>&nbsp;</td>
+                <td>&nbsp;</td>
+            </tr>
+        </table>
+
+
+        <div>&nbsp;</div>
+        <script language="javascript" type="text/javascript">
+
+            /* level - 0:Summary; 1:Failed; 2:All */
+            function showCase(level) {
+            var trs = document.getElementsByTagName("tr");
+            for (var i = 0; i < trs.length; i++){
+                switch(trs[i].id.substr(0, 2)) {
+                    case "ft":
+                    trs[i].className = (level < 1) ? "hiddenRow" : "";
+                    break
+                    case "pt":
+                    trs[i].className = (level > 1) ? "" : "hiddenRow";
+                    break
+                }
+            }
+            }
+
+            function getById(id){
+                return document.getElementById(id);
+            }
+
+            function showClassDetail(cid, count) {
+                var id_list = Array(count);
+                var toHide = 1;
+                for (var i = 0; i < count; i++) {
+                    var tid0 = 't' + cid.substr(1) + '.' + (i+1);
+                    var tr = getById(tid);
+                    var tid = (tr) ? 'f' + tid0 : 'p' + tid0;
+                    id_list[i] = tid;
+                    if (tr.className) {
+                        toHide = 0;
+                    }
+                }
+                for (var i = 0; i < count; i++) {
+                    tid = id_list[i];
+                    if (toHide) {
+                        getById('div_'+tid).style.display = 'none'
+                        getById(tid).className = 'hiddenRow';
+                    }
+                    else {
+                        getById(tid).className = '';
+                    }
+                }
+            }
+
+            function showTestDetail(div_id){
+                var div = getById(div_id);
+                div.style.display = (div.style.display != "block") ? "block" : "none";
+            }
+
+            function html_escape(s) {
+                return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+            }
+
+        </script>
+
+    </body>
+</html>

tests/cmd/commands/test_verify.py

@@ -13,11 +13,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import os
+
 import mock
 import six
 
 from rally.cmd.commands import verify
 from rally import consts
+from rally import exceptions
 from rally import objects
 from tests import test
 
@@ -114,8 +117,60 @@ class VerifyCommandsTestCase(test.TestCase):
                      six.itervalues(tests.data['test_cases']))
         self.verify.show(verification_id)
         mock_print_list.assert_any_call(
-                [verification], fields=total_fields,
+                [verification], fields=total_fields)
-                sortby_index=total_fields.index('Created at'))
         mock_verification_get.assert_called_once_with(verification_id)
         mock_verification_result_get.assert_called_once_with(verification_id)
         mock_print_list.assert_any_call(values, fields, sortby_index=0)
+
+    @mock.patch('rally.db.verification_result_get', return_value={'data': {}})
+    @mock.patch('json.dumps')
+    def test_results(self, mock_json_dumps, mock_db_result_get):
+        verification_uuid = 'a0231bdf-6a4e-4daf-8ab1-ae076f75f070'
+        self.verify.results(verification_uuid, output_json=True)
+
+        mock_db_result_get.assert_called_once_with(verification_uuid)
+        mock_json_dumps.assert_called_once_with({})
+
+    @mock.patch('rally.db.verification_result_get')
+    def test_results_verification_not_found(self, mock_db_result_get):
+        verification_uuid = '9044ced5-9c84-4666-8a8f-4b73a2b62acb'
+        mock_db_result_get.side_effect = exceptions.NotFoundException()
+        self.assertEqual(self.verify.results(verification_uuid), 1)
+
+        mock_db_result_get.assert_called_once_with(verification_uuid)
+
+    @mock.patch('rally.db.verification_result_get', return_value={'data': {}})
+    def test_results_with_output_json_and_output_file(self,
+                                                      mock_db_result_get):
+        verification_uuid = '94615cd4-ff45-4123-86bd-4b0741541d09'
+        self.verify.results(verification_uuid, output_file='results',
+                            output_json=True)
+
+        mock_db_result_get.assert_called_once_with(verification_uuid)
+        self.assertTrue(os.path.isfile('results'))
+
+    @mock.patch('rally.db.verification_result_get', return_value={'data': {}})
+    def test_results_with_output_pprint_and_output_file(self,
+                                                        mock_db_result_get):
+        verification_uuid = 'fa882ccc-153e-4a6e-9001-91fecda6a75c'
+        self.verify.results(verification_uuid, output_pprint=True,
+                            output_file='results')
+
+        mock_db_result_get.assert_called_once_with(verification_uuid)
+        self.assertTrue(os.path.isfile('results'))
+
+    @mock.patch('rally.db.verification_result_get')
+    @mock.patch('rally.verification.verifiers.tempest.json2html.main',
+                return_value='')
+    def test_results_with_output_html_and_output_file(self,
+                                                      mock_json2html_main,
+                                                      mock_db_result_get):
+        verification_uuid = '7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa'
+        results = {'data': {}}
+        mock_db_result_get.return_value = results
+        self.verify.results(verification_uuid, output_html=True,
+                            output_file='results')
+
+        mock_db_result_get.assert_called_once_with(verification_uuid)
+        mock_json2html_main.assert_called_once()
+        self.assertTrue(os.path.isfile('results'))

tests/verification/verifiers/test_json2html.py

@@ -0,0 +1,51 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from rally.verification.verifiers.tempest import json2html
+from tests import test
+
+
+class Json2HtmlTestCase(test.TestCase):
+
+    def test_main(self):
+
+        data = {'tests': 4, 'skipped': 1, 'errors': 1, 'failures': 1,
+                'success': 1, 'time': 22,
+                'test_cases': {
+                    'tp': {'name': 'tp', 'time': 2, 'status': 'OK',
+                           'output': 'tp_ok'},
+                    'ts': {'name': 'ts', 'time': 4, 'status': 'SKIP',
+                           'output': 'ts_skip'},
+                    'tf': {'name': 'tf', 'time': 6, 'status': 'FAIL',
+                           'output': 'tf_fail',
+                           'failure': {'type': 'tf', 'log': 'fail_log'}},
+                    'te': {'name': 'te', 'time': 2, 'status': 'ERROR',
+                           'output': 'te_error',
+                           'failure': {'type': 'te', 'log': 'error+log'}}}}
+
+        obj = json2html.HtmlOutput(data)
+        self.assertEqual(obj.success_count, data['success'])
+        self.assertEqual(obj.failure_count, data['failures'])
+        self.assertEqual(obj.skip_count, data['skipped'])
+        self.assertEqual(obj.error_count, data['errors'])
+
+        report_attrs = obj._getReportAttributes()
+        generator = 'json2html %s' % json2html.__version__
+        heading = obj._generate_heading(report_attrs)
+        report = obj._generate_report()
+        with mock.patch('mako.template.Template') as mock_mako:
+            obj.create_report()
+            mock_mako().render.assert_called_once_with(
+                title=json2html.DEFAULT_TITLE, generator=generator,
+                heading=heading, report=report)