Merge "Add ability to compare two verification results"

rally/cmd/api.py

@@ -32,7 +32,7 @@ LOG = log.getLogger(__name__)
 
 
 def main():
-    # Initialize configuation and logging.
+    # Initialize configuration and logging.
     CONF(sys.argv[1:], project='rally')
     log.setup('rally')
     # Prepare application and bind to the service socket.

rally/cmd/commands/verify.py

@@ -15,6 +15,7 @@
 
 """ Rally command: verify """
 
+import csv
 import json
 import os
 
@@ -30,6 +31,7 @@ from rally.i18n import _
 from rally import objects
 from rally.openstack.common import cliutils as common_cliutils
 from rally.orchestrator import api
+from rally.verification.verifiers.tempest import diff
 from rally.verification.verifiers.tempest import json2html
 
 
@@ -71,30 +73,30 @@ class VerifyCommands(object):
             return (1)
         verification = api.verify(deploy_id, set_name, regex, tempest_config)
         if do_use:
-            use.UseCommands().verification(verification['uuid'])
+            use.UseCommands().verification(verification["uuid"])
 
     def list(self):
         """Display all verifications table, started and finished."""
 
-        fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', 'Failures',
+        fields = ["UUID", "Deployment UUID", "Set name", "Tests", "Failures",
-                  'Created at', 'Status']
+                  "Created at", "Status"]
         verifications = db.verification_list()
         if verifications:
             common_cliutils.print_list(verifications, fields,
-                                       sortby_index=fields.index('Created at'))
+                                       sortby_index=fields.index("Created at"))
         else:
             print(_("There are no results from verifier. To run a verifier, "
                     "use:\nrally verify start"))
 
-    @cliutils.args('--uuid', type=str, dest='verification_uuid',
+    @cliutils.args("--uuid", type=str, dest="verification_uuid",
-                   help='UUID of the verification')
+                   help="UUID of the verification")
-    @cliutils.args('--html', action='store_true', dest='output_html',
+    @cliutils.args("--html", action="store_true", dest="output_html",
-                   help=('Results will be in html format'))
+                   help=("Results will be in html format"))
-    @cliutils.args('--json', action='store_true', dest='output_json',
+    @cliutils.args("--json", action="store_true", dest="output_json",
-                   help=('Results will be in json format'))
+                   help=("Results will be in json format"))
-    @cliutils.args('--output-file', type=str, required=False,
+    @cliutils.args("--output-file", type=str, required=False,
-                   dest='output_file',
+                   dest="output_file",
-                   help='If specified, output will be saved to given file')
+                   help="If specified, output will be saved to given file")
     @envutils.with_default_verification_id
     def results(self, verification_uuid=None, output_file=None,
                 output_html=None, output_json=None):
@@ -107,13 +109,13 @@ class VerifyCommands(object):
         """
 
         try:
-            results = db.verification_result_get(verification_uuid)['data']
+            results = db.verification_result_get(verification_uuid)["data"]
         except exceptions.NotFoundException as e:
             print(six.text_type(e))
             return 1
 
-        result = ''
+        result = ""
-        if len(filter(lambda x: bool(x), [output_json, output_html])) > 1:
+        if output_json + output_html > 1:
             print("Please specify only one output format.")
         elif output_html:
             result = json2html.main(results)
@@ -122,24 +124,24 @@ class VerifyCommands(object):
 
         if output_file:
             output_file = os.path.expanduser(output_file)
-            with open(output_file, 'wb') as f:
+            with open(output_file, "wb") as f:
                 f.write(result)
         else:
             print(result)
 
-    @cliutils.args('--uuid', dest='verification_uuid', type=str,
+    @cliutils.args("--uuid", dest="verification_uuid", type=str,
                    required=False,
-                   help='UUID of a verification')
+                   help="UUID of a verification")
-    @cliutils.args('--sort-by', dest='sort_by', type=str, required=False,
+    @cliutils.args("--sort-by", dest="sort_by", type=str, required=False,
-                   help='Tests can be sorted by "name" or "duration"')
+                   help="Tests can be sorted by 'name' or 'duration'")
-    @cliutils.args('--detailed', dest='detailed', action='store_true',
+    @cliutils.args("--detailed", dest="detailed", action="store_true",
-                   required=False, help='Prints traceback of failed tests')
+                   required=False, help="Prints traceback of failed tests")
     @envutils.with_default_verification_id
-    def show(self, verification_uuid=None, sort_by='name', detailed=False):
+    def show(self, verification_uuid=None, sort_by="name", detailed=False):
         """Display results table of the verification."""
 
         try:
-            sortby_index = ('name', 'duration').index(sort_by)
+            sortby_index = ("name", "duration").index(sort_by)
         except ValueError:
             print("Sorry, but verification results can't be sorted "
                   "by '%s'." % sort_by)
@@ -153,42 +155,104 @@ class VerifyCommands(object):
             return 1
 
         print ("Total results of verification:\n")
-        total_fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests',
+        total_fields = ["UUID", "Deployment UUID", "Set name", "Tests",
-                        'Failures', 'Created at', 'Status']
+                        "Failures", "Created at", "Status"]
         common_cliutils.print_list([verification], fields=total_fields)
 
         print ("\nTests:\n")
-        fields = ['name', 'time', 'status']
+        fields = ["name", "time", "status"]
 
         values = map(objects.Verification,
-                     six.itervalues(tests.data['test_cases']))
+                     six.itervalues(tests.data["test_cases"]))
         common_cliutils.print_list(values, fields, sortby_index=sortby_index)
 
         if detailed:
-            for test in six.itervalues(tests.data['test_cases']):
+            for test in six.itervalues(tests.data["test_cases"]):
-                if test['status'] == 'FAIL':
+                if test["status"] == "FAIL":
                     formatted_test = (
-                        '====================================================='
+                        "====================================================="
-                        '=================\n'
+                        "=================\n"
-                        'FAIL: %(name)s\n'
+                        "FAIL: %(name)s\n"
-                        'Time: %(time)s\n'
+                        "Time: %(time)s\n"
-                        'Type: %(type)s\n'
+                        "Type: %(type)s\n"
-                        '-----------------------------------------------------'
+                        "-----------------------------------------------------"
-                        '-----------------\n'
+                        "-----------------\n"
-                        '%(log)s\n'
+                        "%(log)s\n"
                     ) % {
-                        'name': test['name'], 'time': test['time'],
+                        "name": test["name"], "time": test["time"],
-                        'type': test['failure']['type'],
+                        "type": test["failure"]["type"],
-                        'log': test['failure']['log']}
+                        "log": test["failure"]["log"]}
                     print (formatted_test)
 
-    @cliutils.args('--uuid', dest='verification_uuid', type=str,
+    @cliutils.args("--uuid", dest="verification_uuid", type=str,
                    required=False,
-                   help='UUID of a verification')
+                   help="UUID of a verification")
-    @cliutils.args('--sort-by', dest='sort_by', type=str, required=False,
+    @cliutils.args("--sort-by", dest="sort_by", type=str, required=False,
-                   help='Tests can be sorted by "name" or "duration"')
+                   help="Tests can be sorted by 'name' or 'duration'")
     @envutils.with_default_verification_id
-    def detailed(self, verification_uuid=None, sort_by='name'):
+    def detailed(self, verification_uuid=None, sort_by="name"):
         """Display results table of verification with detailed errors."""
 
         self.show(verification_uuid, sort_by, True)
+
+    @cliutils.args("--uuid-1", type=str, dest="uuid1",
+                   help="UUID of the first verification")
+    @cliutils.args("--uuid-2", type=str, dest="uuid2",
+                   help="UUID of the second verification")
+    @cliutils.args("--csv", action="store_true", dest="output_csv",
+                   help=("Save results in csv format to specified file"))
+    @cliutils.args("--html", action="store_true", dest="output_html",
+                   help=("Save results in html format to specified file"))
+    @cliutils.args("--json", action="store_true", dest="output_json",
+                   help=("Save results in json format to specified file"))
+    @cliutils.args("--output-file", type=str, required=False,
+                   dest="output_file",
+                   help="If specified, output will be saved to given file")
+    @cliutils.args("--threshold", type=int, required=False,
+                   dest="threshold", default=0,
+                   help="If specified, timing differences must exceed this "
+                   "percentage threshold to be included in output")
+    def compare(self, uuid1=None, uuid2=None,
+                output_file=None, output_csv=None, output_html=None,
+                output_json=None, threshold=0):
+        """Compare two verification results.
+
+        :param uuid1: First Verification UUID
+        :param uuid2: Second Verification UUID
+        :param output_file: If specified, output will be saved to given file
+        :param output_csv: Save results in csv format to the specified file
+        :param output_html: Save results in html format to the specified file
+        :param output_json: Save results in json format to the specified file
+                            (Default)
+        :param threshold: Timing difference threshold percentage
+        """
+
+        try:
+            results1 = db.verification_result_get(uuid1)["data"]["test_cases"]
+            results2 = db.verification_result_get(uuid2)["data"]["test_cases"]
+            _diff = diff.Diff(results1, results2, threshold)
+        except exceptions.NotFoundException as e:
+            print(six.text_type(e))
+            return 1
+
+        result = ""
+        if output_json + output_html + output_csv > 1:
+            print("Please specify only one output format, either --json, "
+                  "--html or --csv.")
+            return 1
+        elif output_html:
+            result = _diff.to_html()
+        elif output_csv:
+            result = _diff.to_csv()
+        else:
+            result = _diff.to_json()
+
+        if output_file:
+            with open(output_file, "wb") as f:
+                if output_csv:
+                    writer = csv.writer(f, dialect="excel")
+                    writer.writerows(result)
+                else:
+                    f.write(result)
+        else:
+            print(result)

rally/verification/verifiers/tempest/compare2html.py

@@ -0,0 +1,41 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Output verification comparison results in html."""
+
+import os
+
+import mako.template
+
+__description__ = "List differences between two verification runs"
+__title__ = "Verification Comparison"
+__version__ = "0.1"
+
+
+def create_report(results):
+    template_kw = {
+        "heading": {
+            "title": __title__,
+            "description": __description__,
+            "parameters": [("Difference Count", len(results))]
+        },
+        "generator": "compare2html %s" % __version__,
+        "results": results
+    }
+
+    template_path = os.path.join(os.path.dirname(__file__),
+                                 "report_templates",
+                                 "compare.mako")
+
+    with open(template_path) as f:
+        template = mako.template.Template(f.read(), strict_undefined=True)
+        output = template.render(**template_kw)
+        return output.encode('utf8')

rally/verification/verifiers/tempest/diff.py

@@ -0,0 +1,108 @@
+# Copyright 2014 Dell Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+
+import compare2html
+
+
+class Diff(object):
+
+    def __init__(self, test_cases1, test_cases2, threshold):
+        """Compare two verification results.
+
+        Compares two verification results and emits
+        desired output, csv, html, json or pprint.
+
+        :param test_cases1: older verification json
+        :param test_cases2: newer verification json
+        :param threshold: test time difference percentage threshold
+
+        """
+        self.threshold = threshold
+        self.diffs = self._compare(test_cases1, test_cases2)
+
+    def _compare(self, tc1, tc2):
+        """Compare two verification results.
+
+        :param tc1: first verification test cases json
+        :param tc2: second verification test cases json
+
+        Typical test case json schema:
+            "test_case_key": {
+                "failure": {
+                    "log": ""
+                },
+                "name": "",
+                "output": "",
+                "status": "",
+                "time": 0.0
+            }
+        """
+        names1 = sorted(tc1)
+        names2 = sorted(tc2)
+
+        diffs = []
+        i = j = 0
+        while i < len(names1) and j < len(names2):
+            name1 = names1[i] if i < len(names1) else None
+            name2 = names2[j] if j < len(names2) else None
+            if name1 and name2 and name1 == name2:
+                diffs.extend(self._diff_values(name1, tc1[name1], tc2[name2]))
+                i += 1
+                j += 1
+
+            elif (not name1) or (name1 > name2):
+                diffs.append({"type": "new_test", "test_name": name2})
+                j += 1
+            else:
+                diffs.append({"type": "removed_test", "test_name": name1})
+                i += 1
+
+        return diffs
+
+    def _diff_values(self, name, result1, result2):
+        th = self.threshold
+        fields = ["status", "time", "output"]
+        diffs = []
+        for field in fields:
+            val1 = result1[field]
+            val2 = result2[field]
+            if val1 != val2 and not (field == "time"
+                                     and abs(((val2 - val1) / val1) * 100)
+                                     < th):
+                diffs.append({
+                    "field": field,
+                    "type": "value_changed",
+                    "test_name": name,
+                    "val1": val1,
+                    "val2": val2
+                })
+        return diffs
+
+    def to_csv(self):
+        rows = (("Type", "Field", "Value 1", "Value 2", "Test Name"),)
+        for res in self.diffs:
+            row = (res.get("type"), res.get("field", ""),
+                   res.get("val1", ""), res.get("val2", ""),
+                   res.get("test_name"))
+            rows = rows + (row,)
+        return rows
+
+    def to_json(self):
+        return json.dumps(self.diffs, sort_keys=True, indent=4)
+
+    def to_html(self):
+        return compare2html.create_report(self.diffs)

rally/verification/verifiers/tempest/report_templates/compare.mako

@@ -0,0 +1,165 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!doctype html>
+<html>
+    <head>
+        <title>${heading["title"]}</title>
+        <meta name="generator" content="${generator}">
+        <meta charset="utf-8">
+        <script type="text/javascript">
+            var DOWN = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAGCAYAAAAVMmT4AAAAJUlEQVQYlWNgYGD4TwJmYCBFIYYGFhYWvArx2YAXEK0QWQMGAADd8SPpeGzm9QAAAABJRU5ErkJggg==";
+            var NONE = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAGCAYAAAAVMmT4AAAADUlEQVQYlWNgGAUIAAABDgAB6WzgmwAAAABJRU5ErkJggg==";
+            var UP = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAGCAYAAAAVMmT4AAAAK0lEQVQYlWNgwA7+4xDHqhCGiVaIVwNcAQsLC14N2EzEqoEYhf8ZGBj+AwCZbyPp8zIdEAAAAABJRU5ErkJggg==";
+
+            function sort_table(table_id, col, sort){
+                var table = document.getElementById(table_id);
+                var tbody = table.tBodies[0];
+                var header_row = table.tHead.rows[0];
+                render_header(col, sort, header_row);
+                sort_results(tbody, col, sort);
+            }
+
+            function render_header(col, sort, header_row){
+                var h_cells = header_row.cells;
+                for(i = 0; i < h_cells.length; i++){
+                    var cell = h_cells[i];
+                    var img = cell.firstElementChild;
+                    if (i == col){
+                        if (sort == 1){
+                            img.src = UP;
+                        }else{
+                            img.src = DOWN;
+                        }
+                    }else{ //spacer image
+                        img.src = NONE;
+                    }
+                }
+            }
+
+            function sort_results(tbody, col, sort) {
+                var rows = tbody.rows, rlen = rows.length, arr = new Array(), i, j, cells, clen;
+                // fill the array with values from the table
+                for(i = 0; i < rlen; i++){
+                    cells = rows[i].cells;
+                    clen = cells.length;
+                    arr[i] = new Array();
+                    for(j = 0; j < clen; j++){
+                        arr[i][j] = cells[j].innerHTML;
+                    }
+                }
+                // sort the array by the specified column number (col) and order (sort)
+                arr.sort(function(a, b){
+                    return (a[col] == b[col]) ? 0 : ((a[col] > b[col]) ? sort : -1*sort);
+                });
+                for(i = 0; i < rlen; i++){
+                    arr[i] = "<td>"+arr[i].join("</td><td>")+"</td>";
+                }
+                tbody.innerHTML = "<tr>"+arr.join("</tr><tr>")+"</tr>";
+            }
+        </script>
+        <style type="text/css" media="screen">
+            body {
+                font-family: verdana, arial, helvetica, sans-serif;
+                font-size: 80%;
+            }
+
+            table {
+                font-size: 100%; width: 100%;
+            }
+
+            h1 {
+                font-size: 16pt;
+                color: gray;
+            }
+
+            .heading {
+                margin-top: 0ex;
+                margin-bottom: 1ex;
+            }
+
+            .heading .attribute {
+                margin-top: 1ex;
+                margin-bottom: 0;
+            }
+
+            .heading .description {
+                margin-top: 4ex;
+                margin-bottom: 6ex;
+            }
+
+            #results_table {
+                width: 100%;
+                border-collapse: collapse;
+                border: 1px solid #777;
+            }
+
+            #header_row {
+                font-weight: bold;
+                color: white;
+                background-color: #777;
+            }
+
+            #results_table td {
+                border: 1px solid #777;
+                padding: 2px;
+            }
+
+            .testcase   { margin-left: 2em;}
+
+            img.updown{
+                padding-left: 3px;
+                padding-bottom: 2px;
+            }
+
+            th:hover{
+                cursor:pointer;
+            }
+
+            .nowrap {white-space: nowrap;}
+
+        </style>
+    </head>
+
+    <body>
+        <div class="heading">
+            <h1>${heading["title"]}</h1>
+            % for name, value in heading["parameters"]:
+                <p class="attribute"><strong>${name}:</strong> ${value}</p>
+                % endfor
+                <p class="description">${heading["description"]}</p>
+        </div>
+
+        <table id="results_table">
+            <colgroup>
+            <col align="left" />
+            <col align="left" />
+            <col align="left" />
+            <col align="left" />
+            <col align="left" />
+            </colgroup>
+            <thead>
+            <tr id="header_row">
+                <th class="nowrap" onclick="sort_table('results_table', 0, col1_sort); col1_sort *= -1; col2_sort = 1; col3_sort = 1; col4_sort = 1; col5_sort = 1;">Type<img class="updown" src=NONE /></th>
+                <th class="nowrap" onclick="sort_table('results_table', 1, col2_sort); col2_sort *= -1; col1_sort = 1; col3_sort = 1; col4_sort = 1; col5_sort = 1;">Field<img class="updown"  src=NONE /></th>
+                <th class="nowrap" onclick="sort_table('results_table', 2, col3_sort); col3_sort *= -1; col1_sort = 1; col2_sort = 1; col4_sort = 1; col5_sort = 1;">Value 1<img class="updown" src=NONE /></th>
+                <th class="nowrap" onclick="sort_table('results_table', 3, col4_sort); col4_sort *= -1; col1_sort = 1; col2_sort = 1; col3_sort = 1; col5_sort = 1;">Value 2<img class="updown" src=NONE /></th>
+                <th onclick="sort_table('results_table', 4, col5_sort); col5_sort *= -1; col1_sort = 1; col2_sort = 1; col3_sort = 1; col4_sort = 1;">Test Name<img class="updown" src=NONE /></th>
+            </tr>
+            </thead>
+            <tbody id="results">
+            % for diff in results:
+                <tr class="">
+                    <td class="type">${diff.get("type")}</td>
+                    <td class="field">${diff.get("field", "")}</td>
+                    <td class="val">${diff.get("val1", "")}</td>
+                    <td class="val">${diff.get("val2", "")}</td>
+                    <td class="testname">${diff.get("test_name")}</td>
+                </tr>
+            % endfor
+        </table>
+        <script type="text/javascript">
+            var col1_sort = 1, col2_sort = 1, col3_sort = 1; col4_sort = 1; col5_sort = 1;
+            sort_table("results_table", 4, col5_sort);
+            col5_sort *= -1;
+        </script>
+    </body>
+</html>

tests/unit/cmd/commands/test_verify.py

@@ -31,41 +31,41 @@ class VerifyCommandsTestCase(test.TestCase):
         self.verify = verify.VerifyCommands()
 
         self.image1 = mock.Mock()
-        self.image1.name = 'cirros-1'
+        self.image1.name = "cirros-1"
-        self.image1.id = 'fake_image_id_1'
+        self.image1.id = "fake_image_id_1"
         self.image2 = mock.Mock()
-        self.image2.id = 'fake_image_id_2'
+        self.image2.id = "fake_image_id_2"
-        self.image2.name = 'cirros-2'
+        self.image2.name = "cirros-2"
 
         self.flavor1 = mock.Mock()
         self.flavor2 = mock.Mock()
-        self.flavor1.id = 'fake_flavor_id_1'
+        self.flavor1.id = "fake_flavor_id_1"
-        self.flavor2.id = 'fake_flavor_id_2'
+        self.flavor2.id = "fake_flavor_id_2"
         self.flavor1.ram = 128
         self.flavor2.ram = 64
 
-    @mock.patch('rally.osclients.Clients')
+    @mock.patch("rally.osclients.Clients")
-    @mock.patch('rally.orchestrator.api.verify')
+    @mock.patch("rally.orchestrator.api.verify")
     def test_start(self, mock_verify, mock_clients):
-        deploy_id = '0fba91c6-82d5-4ce1-bd00-5d7c989552d9'
+        deploy_id = "0fba91c6-82d5-4ce1-bd00-5d7c989552d9"
         mock_clients().glance().images.list.return_value = [
             self.image1, self.image2]
         mock_clients().nova().flavors.list.return_value = [
             self.flavor1, self.flavor2]
 
         self.verify.start(deploy_id=deploy_id)
-        default_set_name = 'smoke'
+        default_set_name = "smoke"
         default_regex = None
 
         mock_verify.assert_called_once_with(deploy_id,
                                             default_set_name, default_regex,
                                             None)
 
-    @mock.patch('rally.osclients.Clients')
+    @mock.patch("rally.osclients.Clients")
-    @mock.patch('rally.orchestrator.api.verify')
+    @mock.patch("rally.orchestrator.api.verify")
     def test_start_with_user_specified_tempest_config(self, mock_verify,
                                                       mock_clients):
-        deploy_id = '0fba91c6-82d5-4ce1-bd00-5d7c989552d9'
+        deploy_id = "0fba91c6-82d5-4ce1-bd00-5d7c989552d9"
         mock_clients().glance().images.list.return_value = [
             self.image1, self.image2]
         mock_clients().nova().flavors.list.return_value = [
@@ -73,7 +73,7 @@ class VerifyCommandsTestCase(test.TestCase):
         tempest_config = tempfile.NamedTemporaryFile()
         self.verify.start(deploy_id=deploy_id,
                           tempest_config=tempest_config.name)
-        default_set_name = 'smoke'
+        default_set_name = "smoke"
         default_regex = None
 
         mock_verify.assert_called_once_with(deploy_id,
@@ -81,114 +81,212 @@ class VerifyCommandsTestCase(test.TestCase):
                                             tempest_config.name)
         tempest_config.close()
 
-    @mock.patch('rally.orchestrator.api.verify')
+    @mock.patch("rally.orchestrator.api.verify")
     def test_start_with_wrong_set_name(self, mock_verify):
-        deploy_id = 'f2009aae-6ef3-468e-96b2-3c987d584010'
+        deploy_id = "f2009aae-6ef3-468e-96b2-3c987d584010"
 
-        wrong_set_name = 'unexpected_value'
+        wrong_set_name = "unexpected_value"
 
         self.verify.start(deploy_id, wrong_set_name)
 
         self.assertNotIn(wrong_set_name, consts.TEMPEST_TEST_SETS)
         self.assertFalse(mock_verify.called)
 
-    @mock.patch('rally.openstack.common.cliutils.print_list')
+    @mock.patch("rally.openstack.common.cliutils.print_list")
-    @mock.patch('rally.db.verification_list')
+    @mock.patch("rally.db.verification_list")
     def test_list(self, mock_db_verification_list, mock_print_list):
-        fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', 'Failures',
+        fields = ["UUID", "Deployment UUID", "Set name", "Tests", "Failures",
-                  'Created at', 'Status']
+                  "Created at", "Status"]
-        verifications = {'dummy': []}
+        verifications = {"dummy": []}
         mock_db_verification_list.return_value = verifications
         self.verify.list()
         mock_db_verification_list.assert_called_once_with()
         mock_print_list.assert_called_once_with(verifications, fields,
                                                 sortby_index=fields.index(
-                                                    'Created at'))
+                                                    "Created at"))
 
-    @mock.patch('rally.openstack.common.cliutils.print_list')
+    @mock.patch("rally.openstack.common.cliutils.print_list")
-    @mock.patch('rally.db.verification_get')
+    @mock.patch("rally.db.verification_get")
-    @mock.patch('rally.db.verification_result_get')
+    @mock.patch("rally.db.verification_result_get")
-    @mock.patch('rally.objects.Verification')
+    @mock.patch("rally.objects.Verification")
     def test_show(self, mock_obj_verification,
                   mock_verification_result_get, mock_verification_get,
                   mock_print_list):
 
         class Test_dummy():
-            data = {'test_cases': {'test_a': {'name': 'test_a', 'time': 20,
+            data = {"test_cases": {"test_a": {"name": "test_a", "time": 20,
-                                              'status': 'PASS'},
+                                              "status": "PASS"},
-                                   'test_b': {'name': 'test_b', 'time': 20,
+                                   "test_b": {"name": "test_b", "time": 20,
-                                              'status': 'SKIP'},
+                                              "status": "SKIP"},
-                                   'test_c': {'name': 'test_c', 'time': 20,
+                                   "test_c": {"name": "test_c", "time": 20,
-                                              'status': 'FAIL'}}}
+                                              "status": "FAIL"}}}
 
-        verification_id = '39121186-b9a4-421d-b094-6c6b270cf9e9'
+        verification_id = "39121186-b9a4-421d-b094-6c6b270cf9e9"
-        total_fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests',
+        total_fields = ["UUID", "Deployment UUID", "Set name", "Tests",
-                        'Failures', 'Created at', 'Status']
+                        "Failures", "Created at", "Status"]
-        fields = ['name', 'time', 'status']
+        fields = ["name", "time", "status"]
         verification = mock.MagicMock()
         tests = Test_dummy()
         mock_verification_result_get.return_value = tests
         mock_verification_get.return_value = verification
         mock_obj_verification.return_value = 1
         values = map(objects.Verification,
-                     six.itervalues(tests.data['test_cases']))
+                     six.itervalues(tests.data["test_cases"]))
         self.verify.show(verification_id)
-        mock_print_list.assert_any_call(
+        mock_print_list.assert_any_call([verification], fields=total_fields)
-                [verification], fields=total_fields)
         mock_verification_get.assert_called_once_with(verification_id)
         mock_verification_result_get.assert_called_once_with(verification_id)
         mock_print_list.assert_any_call(values, fields, sortby_index=0)
 
-    @mock.patch('rally.db.verification_result_get', return_value={'data': {}})
+    @mock.patch("rally.db.verification_result_get", return_value={"data": {}})
-    @mock.patch('json.dumps')
+    @mock.patch("json.dumps")
     def test_results(self, mock_json_dumps, mock_db_result_get):
-        verification_uuid = 'a0231bdf-6a4e-4daf-8ab1-ae076f75f070'
+        verification_uuid = "a0231bdf-6a4e-4daf-8ab1-ae076f75f070"
-        self.verify.results(verification_uuid, output_json=True)
+        self.verify.results(verification_uuid, output_html=False,
+                            output_json=True)
 
         mock_db_result_get.assert_called_once_with(verification_uuid)
         mock_json_dumps.assert_called_once_with({}, sort_keys=True, indent=4)
 
-    @mock.patch('rally.db.verification_result_get')
+    @mock.patch("rally.db.verification_result_get")
     def test_results_verification_not_found(self, mock_db_result_get):
-        verification_uuid = '9044ced5-9c84-4666-8a8f-4b73a2b62acb'
+        verification_uuid = "9044ced5-9c84-4666-8a8f-4b73a2b62acb"
         mock_db_result_get.side_effect = exceptions.NotFoundException()
-        self.assertEqual(self.verify.results(verification_uuid), 1)
+        self.assertEqual(self.verify.results(verification_uuid,
+                                             output_html=False,
+                                             output_json=True), 1)
 
         mock_db_result_get.assert_called_once_with(verification_uuid)
 
-    @mock.patch('rally.cmd.commands.verify.open', create=True)
+    @mock.patch("rally.cmd.commands.verify.open", create=True)
-    @mock.patch('rally.db.verification_result_get', return_value={'data': {}})
+    @mock.patch("rally.db.verification_result_get", return_value={"data": {}})
     def test_results_with_output_json_and_output_file(self,
                                                       mock_db_result_get,
                                                       mock_open):
         mock_open.return_value = mock.MagicMock()
-        verification_uuid = '94615cd4-ff45-4123-86bd-4b0741541d09'
+        verification_uuid = "94615cd4-ff45-4123-86bd-4b0741541d09"
-        self.verify.results(verification_uuid, output_file='results',
+        self.verify.results(verification_uuid, output_file="results",
-                            output_json=True)
+                            output_html=False, output_json=True)
 
         mock_db_result_get.assert_called_once_with(verification_uuid)
-        mock_open.assert_called_once_with('results', 'wb')
+        mock_open.assert_called_once_with("results", "wb")
         fake_file = mock_open.return_value.__enter__.return_value
-        fake_file.write.assert_called_once_with('{}')
+        fake_file.write.assert_called_once_with("{}")
 
-    @mock.patch('rally.cmd.commands.verify.open', create=True)
+    @mock.patch("rally.cmd.commands.verify.open", create=True)
-    @mock.patch('rally.db.verification_result_get')
+    @mock.patch("rally.db.verification_result_get")
-    @mock.patch('rally.verification.verifiers.tempest.json2html.main',
+    @mock.patch("rally.verification.verifiers.tempest.json2html.main",
-                return_value='')
+                return_value="")
     def test_results_with_output_html_and_output_file(self,
                                                       mock_json2html_main,
                                                       mock_db_result_get,
                                                       mock_open):
         mock_open.return_value = mock.MagicMock()
-        verification_uuid = '7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa'
+        verification_uuid = "7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa"
         fake_data = {}
-        results = {'data': fake_data}
+        results = {"data": fake_data}
         mock_db_result_get.return_value = results
         self.verify.results(verification_uuid, output_html=True,
-                            output_file='results')
+                            output_json=False, output_file="results")
 
         mock_db_result_get.assert_called_once_with(verification_uuid)
         mock_json2html_main.assert_called_once_with(fake_data)
-        mock_open.assert_called_once_with('results', 'wb')
+        mock_open.assert_called_once_with("results", "wb")
         fake_file = mock_open.return_value.__enter__.return_value
-        fake_file.write.assert_called_once_with('')
+        fake_file.write.assert_called_once_with("")
+
+    @mock.patch("rally.db.verification_result_get",
+                return_value={"data": {"test_cases": {}}})
+    @mock.patch("json.dumps")
+    def test_compare(self, mock_json_dumps, mock_db_result_get):
+        uuid1 = "8eda1b10-c8a4-4316-9603-8468ff1d1560"
+        uuid2 = "f6ef0a98-1b18-452f-a6a7-922555c2e326"
+        self.verify.compare(uuid1, uuid2, output_csv=False, output_html=False,
+                            output_json=True)
+
+        fake_data = []
+        calls = [mock.call(uuid1),
+                 mock.call(uuid2)]
+        mock_db_result_get.assert_has_calls(calls, True)
+        mock_json_dumps.assert_called_once_with(fake_data, sort_keys=True,
+                                                indent=4)
+
+    @mock.patch("rally.db.verification_result_get",
+                side_effect=exceptions.NotFoundException())
+    def test_compare_verification_not_found(self, mock_db_result_get):
+        uuid1 = "f7dc82da-31a6-4d40-bbf8-6d366d58960f"
+        uuid2 = "2f8a05f3-d310-4f02-aabf-e1165aaa5f9c"
+
+        self.assertEqual(self.verify.compare(uuid1, uuid2, output_csv=False,
+                                             output_html=False,
+                                             output_json=True), 1)
+
+        mock_db_result_get.assert_called_once_with(uuid1)
+
+    @mock.patch("rally.cmd.commands.verify.open", create=True)
+    @mock.patch("rally.db.verification_result_get",
+                return_value={"data": {"test_cases": {}}})
+    def test_compare_with_output_csv_and_output_file(self,
+                                                     mock_db_result_get,
+                                                     mock_open):
+
+        fake_string = "Type,Field,Value 1,Value 2,Test Name\r\n"
+        uuid1 = "5e744557-4c3a-414f-9afb-7d3d8708028f"
+        uuid2 = "efe1c74d-a632-476e-bb6a-55a9aa9cf76b"
+        self.verify.compare(uuid1, uuid2, output_file="results",
+                            output_csv=True, output_html=False,
+                            output_json=False)
+
+        calls = [mock.call(uuid1),
+                 mock.call(uuid2)]
+        mock_db_result_get.assert_has_calls(calls, True)
+        mock_open.assert_called_once_with("results", "wb")
+        fake_file = mock_open.return_value.__enter__.return_value
+        fake_file.write.assert_called_once_with(fake_string)
+
+    @mock.patch("rally.cmd.commands.verify.open", create=True)
+    @mock.patch("rally.db.verification_result_get",
+                return_value={"data": {"test_cases": {}}})
+    def test_compare_with_output_json_and_output_file(self,
+                                                      mock_db_result_get,
+                                                      mock_open):
+        fake_json_string = "[]"
+        uuid1 = "0505e33a-738d-4474-a611-9db21547d863"
+        uuid2 = "b1908417-934e-481c-8d23-bc0badad39ed"
+        self.verify.compare(uuid1, uuid2, output_file="results",
+                            output_csv=False, output_html=False,
+                            output_json=True)
+
+        calls = [mock.call(uuid1),
+                 mock.call(uuid2)]
+        mock_db_result_get.assert_has_calls(calls, True)
+        mock_open.assert_called_once_with("results", "wb")
+        fake_file = mock_open.return_value.__enter__.return_value
+        fake_file.write.assert_called_once_with(fake_json_string)
+
+    @mock.patch("rally.cmd.commands.verify.open", create=True)
+    @mock.patch("rally.db.verification_result_get")
+    @mock.patch(("rally.verification.verifiers.tempest."
+                 "compare2html.create_report"), return_value="")
+    def test_compare_with_output_html_and_output_file(self,
+                                                      mock_compare2html_create,
+                                                      mock_db_result_get,
+                                                      mock_open):
+
+        uuid1 = "cdf64228-77e9-414d-9d4b-f65e9d62c61f"
+        uuid2 = "39393eec-1b45-4103-8ec1-631edac4b8f0"
+        results = {"data": {"test_cases": {}}}
+        fake_data = []
+        self.verify.compare(uuid1, uuid2,
+                            output_file="results",
+                            output_csv=False, output_html=True,
+                            output_json=False)
+        mock_db_result_get.return_value = results
+        calls = [mock.call(uuid1),
+                 mock.call(uuid2)]
+        mock_db_result_get.assert_has_calls(calls, True)
+        mock_compare2html_create.assert_called_once_with(fake_data)
+
+        mock_open.assert_called_once_with("results", "wb")
+        fake_file = mock_open.return_value.__enter__.return_value
+        fake_file.write.assert_called_once_with("")

tests/unit/verification/verifiers/test_compare2html.py

@@ -0,0 +1,39 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from rally.verification.verifiers.tempest import compare2html
+from tests.unit import test
+
+
+class Compare2HtmlTestCase(test.TestCase):
+
+    def test_main(self):
+        results = [{'val2': 0.0111, 'field': u'time', 'val1': 0.0222,
+                    'type': 'CHANGED', 'test_name': u'test.one'},
+                   {'val2': 0.111, 'field': u'time', 'val1': 0.222,
+                    'type': 'CHANGED', 'test_name': u'test.two'},
+                   {'val2': 1.11, 'field': u'time', 'val1': 2.22,
+                    'type': 'CHANGED', 'test_name': u'test.three'}]
+
+        fake_kw = {"heading":
+                   {"title": compare2html.__title__,
+                    "description": compare2html.__description__,
+                    "parameters": [("Difference Count", len(results))]
+                    },
+                   "generator": "compare2html %s" % compare2html.__version__,
+                   "results": results}
+
+        with mock.patch('mako.template.Template') as mock_mako:
+            compare2html.create_report(results)
+            mock_mako().render.assert_called_once_with(**fake_kw)

tests/unit/verification/verifiers/test_diff.py

@@ -0,0 +1,76 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from rally.verification.verifiers.tempest import diff
+from tests.unit import test
+
+
+class DiffTestCase(test.TestCase):
+
+    def test_main(self):
+        results1 = {'test.NONE': {'name': 'test.NONE',
+                                  'output': 'test.NONE',
+                                  'status': 'SKIPPED',
+                                  'time': 0.000},
+                    'test.one': {'name': 'test.one',
+                                 'output': 'test.one',
+                                 'status': 'OK',
+                                 'time': 0.111},
+                    'test.two': {'name': 'test.two',
+                                 'output': 'test.two',
+                                 'status': 'OK',
+                                 'time': 0.222},
+                    'test.three': {'name': 'test.three',
+                                   'output': 'test.three',
+                                   'status': 'FAILED',
+                                   'time': 0.333},
+                    'test.four': {'name': 'test.four',
+                                  'output': 'test.four',
+                                  'status': 'OK',
+                                  'time': 0.444},
+                    'test.five': {'name': 'test.five',
+                                  'output': 'test.five',
+                                  'status': 'OK',
+                                  'time': 0.555}
+                    }
+
+        results2 = {'test.one': {'name': 'test.one',
+                                 'output': 'test.one',
+                                 'status': 'FAIL',
+                                 'time': 0.1111},
+                    'test.two': {'name': 'test.two',
+                                 'output': 'test.two',
+                                 'status': 'OK',
+                                 'time': 0.222},
+                    'test.three': {'name': 'test.three',
+                                   'output': 'test.three',
+                                   'status': 'OK',
+                                   'time': 0.3333},
+                    'test.four': {'name': 'test.four',
+                                  'output': 'test.four',
+                                  'status': 'FAIL',
+                                  'time': 0.4444},
+                    'test.five': {'name': 'test.five',
+                                  'output': 'test.five',
+                                  'status': 'OK',
+                                  'time': 0.555},
+                    'test.six': {'name': 'test.six',
+                                 'output': 'test.six',
+                                 'status': 'OK',
+                                 'time': 0.666}
+                    }
+
+        diff_ = diff.Diff(results1, results2, 0)
+        assert len(diff_.diffs) == 8
+        assert diff_.to_csv() != ''
+        assert diff_.to_html() != ''
+        assert diff_.to_json() != ''

tools/rally.bash_completion

@@ -31,6 +31,7 @@ _rally()
     OPTS["show_keypairs"]="--deploy-id"
     OPTS["show_networks"]="--deploy-id"
     OPTS["show_secgroups"]="--deploy-id"
+    OPTS["verify_compare"]="--uuid-1 --uuid-2 --csv --html --json --output-file --threshold"
     OPTS["verify_detailed"]="--uuid --sort-by"
     OPTS["verify_list"]=""
     OPTS["verify_results"]="--uuid --html --json --output-file"
@@ -74,4 +75,4 @@ _rally()
     fi
     return 0
 }
-complete -F _rally rally
+complete -F _rally rally