[Reports] Significant improvements in verification report
* ability to show several verifications results (this
deprecates command `rally verify compare')
* command `rally verify compare' is removed - new report
compares results in better way
* new, unified, AngularJS/Jinja2-based template
* new verifications results processing
* new module rally.ui.report which cares about
report generation (this is a place where code from
rally.task.processing.plot should be also moved)
Examples:
rally verify results\
--uuid <uuid> --html > single_verification_result.html
rally verify results\
--uuid <uuid1> <uuid2> <uuid3> --html > compare_3.html
Co-Authored-By: Oleksandr Savatieiev <osavatieiev@mirantis.com>
Co-Authored-By: Alexander Maretskiy <amaretskiy@mirantis.com>
Change-Id: I942e0d9bf2094f3254dbeccbaa76dbbc3a3ca40e
This commit is contained in:
Oleksandr Savatieiev
Alexander Maretskiy
parent
d599de9a26
commit
70901577ab
@ -46,7 +46,7 @@ _rally()
|
||||
OPTS["task_trends"]="--out --open --tasks"
|
||||
OPTS["task_use"]="--uuid"
|
||||
OPTS["task_validate"]="--deployment --task --task-args --task-args-file"
|
||||
OPTS["verify_compare"]="--uuid-1 --uuid-2 --csv --html --json --output-file --threshold"
|
||||
OPTS["verify_compare"]=""
|
||||
OPTS["verify_detailed"]="--uuid --sort-by"
|
||||
OPTS["verify_discover"]="--deployment --pattern --system-wide"
|
||||
OPTS["verify_genconfig"]="--deployment --tempest-config --add-options --override"
|
||||
@ -56,7 +56,7 @@ _rally()
|
||||
OPTS["verify_list"]=""
|
||||
OPTS["verify_listplugins"]="--deployment --system-wide"
|
||||
OPTS["verify_reinstall"]="--deployment --source --version --system-wide"
|
||||
OPTS["verify_results"]="--uuid --html --json --output-file"
|
||||
OPTS["verify_results"]="--uuid --html --json --csv --output-file"
|
||||
OPTS["verify_show"]="--uuid --sort-by --detailed"
|
||||
OPTS["verify_showconfig"]="--deployment"
|
||||
OPTS["verify_start"]="--deployment --set --regex --load-list --skip-list --tempest-config --xfail-list --no-use --system-wide --concurrency --failing"
|
||||
|
||||
@ -15,8 +15,8 @@
|
||||
|
||||
"""Rally command: verify"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
|
||||
import six
|
||||
@ -31,9 +31,7 @@ from rally.common.i18n import _
|
||||
from rally.common import utils
|
||||
from rally import consts
|
||||
from rally import exceptions
|
||||
from rally.verification.tempest import diff
|
||||
from rally.verification.tempest import json2html
|
||||
|
||||
from rally.ui import report
|
||||
|
||||
AVAILABLE_SETS = list(consts.TempestTestsSets) + list(consts.TempestTestsAPI)
|
||||
|
||||
@ -223,40 +221,88 @@ class VerifyCommands(object):
|
||||
print(_("No verification was started yet. "
|
||||
"To start verification use:\nrally verify start"))
|
||||
|
||||
@cliutils.args("--uuid", type=str, dest="verification",
|
||||
help="UUID of a verification.")
|
||||
@cliutils.args("--uuid", nargs="+", dest="uuids",
|
||||
help="UUIDs of verifications.")
|
||||
@cliutils.args("--html", action="store_true", dest="output_html",
|
||||
help="Display results in HTML format.")
|
||||
@cliutils.args("--json", action="store_true", dest="output_json",
|
||||
help="Display results in JSON format.")
|
||||
@cliutils.args("--csv", action="store_true", dest="output_csv",
|
||||
help="Display results in CSV format")
|
||||
@cliutils.args("--output-file", type=str, required=False,
|
||||
dest="output_file", metavar="<path>",
|
||||
help="Path to a file to save results to.")
|
||||
@envutils.with_default_verification_id
|
||||
@cliutils.suppress_warnings
|
||||
def results(self, verification=None, output_file=None,
|
||||
output_html=None, output_json=None):
|
||||
def results(self, uuids=None, output_file=None,
|
||||
output_html=False, output_json=False, output_csv=False):
|
||||
"""Display results of a verification.
|
||||
|
||||
:param verification: UUID of a verification
|
||||
:param output_file: Path to a file to save results
|
||||
:param output_html: Display results in HTML format
|
||||
:param output_json: Display results in JSON format (Default)
|
||||
:param output_html: Display results in HTML format
|
||||
:param output_csv: Display results in CSV format
|
||||
"""
|
||||
if not uuids:
|
||||
uuid = envutils.get_global(envutils.ENV_VERIFICATION)
|
||||
if not uuid:
|
||||
raise exceptions.InvalidArgumentsException(
|
||||
"Verification UUID is missing")
|
||||
uuids = [uuid]
|
||||
data = []
|
||||
for uuid in uuids:
|
||||
try:
|
||||
results = api.Verification.get(verification).get_results()
|
||||
verification = api.Verification.get(uuid)
|
||||
except exceptions.NotFoundException as e:
|
||||
print(six.text_type(e))
|
||||
return 1
|
||||
data.append(verification)
|
||||
|
||||
result = ""
|
||||
if output_json + output_html > 1:
|
||||
print(_("Please specify only one "
|
||||
"output format: --json or --html."))
|
||||
elif output_html:
|
||||
result = json2html.generate_report(results)
|
||||
if output_json + output_html + output_csv > 1:
|
||||
print(_("Please specify only one format option from %s.")
|
||||
% "--json, --html, --csv")
|
||||
return 1
|
||||
|
||||
verifications = {}
|
||||
for ver in data:
|
||||
uuid = ver.db_object["uuid"]
|
||||
res = ver.get_results() or {}
|
||||
tests = {}
|
||||
for test in list(res.get("test_cases", {}).values()):
|
||||
name = test["name"]
|
||||
if name in tests:
|
||||
mesg = ("Duplicated test in verification "
|
||||
"%(uuid)s: %(test)s" % {"uuid": uuid,
|
||||
"test": name})
|
||||
raise exceptions.RallyException(mesg)
|
||||
tests[name] = {"tags": test["tags"],
|
||||
"status": test["status"],
|
||||
"duration": test["time"],
|
||||
"details": (test.get("traceback", "").strip()
|
||||
or test.get("reason"))}
|
||||
verifications[uuid] = {
|
||||
"tests": tests,
|
||||
"duration": res.get("time", 0),
|
||||
"total": res.get("tests", 0),
|
||||
"skipped": res.get("skipped", 0),
|
||||
"success": res.get("success", 0),
|
||||
"expected_failures": res.get("expected_failures", 0),
|
||||
"unexpected_success": res.get("unexpected_success", 0),
|
||||
"failures": res.get("failures", 0),
|
||||
"started_at": ver.db_object[
|
||||
"created_at"].strftime("%Y-%d-%m %H:%M:%S"),
|
||||
"finished_at": ver.db_object[
|
||||
"updated_at"].strftime("%Y-%d-%m %H:%M:%S"),
|
||||
"status": ver.db_object["status"],
|
||||
"set_name": ver.db_object["set_name"]
|
||||
}
|
||||
|
||||
if output_html:
|
||||
result = report.VerificationReport(verifications).to_html()
|
||||
elif output_csv:
|
||||
result = report.VerificationReport(verifications).to_csv()
|
||||
else:
|
||||
result = json.dumps(results, sort_keys=True, indent=4)
|
||||
result = report.VerificationReport(verifications).to_json()
|
||||
|
||||
if output_file:
|
||||
output_file = os.path.expanduser(output_file)
|
||||
@ -328,67 +374,13 @@ class VerifyCommands(object):
|
||||
"""
|
||||
self.show(verification, sort_by, True)
|
||||
|
||||
@cliutils.args("--uuid-1", type=str, required=True, dest="verification1",
|
||||
help="UUID of the first verification")
|
||||
@cliutils.args("--uuid-2", type=str, required=True, dest="verification2",
|
||||
help="UUID of the second verification")
|
||||
@cliutils.args("--csv", action="store_true", dest="output_csv",
|
||||
help="Display results in CSV format")
|
||||
@cliutils.args("--html", action="store_true", dest="output_html",
|
||||
help="Display results in HTML format")
|
||||
@cliutils.args("--json", action="store_true", dest="output_json",
|
||||
help="Display results in JSON format")
|
||||
@cliutils.args("--output-file", type=str, required=False,
|
||||
dest="output_file", help="Path to a file to save results")
|
||||
@cliutils.args("--threshold", type=int, required=False,
|
||||
dest="threshold", default=0,
|
||||
help="If specified, timing differences must exceed this "
|
||||
"percentage threshold to be included in output")
|
||||
def compare(self, verification1=None, verification2=None,
|
||||
output_file=None, output_csv=None, output_html=None,
|
||||
output_json=None, threshold=0):
|
||||
"""Compare two verification results.
|
||||
|
||||
:param verification1: UUID of the first verification
|
||||
:param verification2: UUID of the second verification
|
||||
:param output_file: Path to a file to save results
|
||||
:param output_csv: Display results in CSV format
|
||||
:param output_html: Display results in HTML format
|
||||
:param output_json: Display results in JSON format (Default)
|
||||
:param threshold: Timing difference threshold percentage
|
||||
"""
|
||||
try:
|
||||
res_1 = api.Verification.get(
|
||||
verification1).get_results()["test_cases"]
|
||||
res_2 = api.Verification.get(
|
||||
verification2).get_results()["test_cases"]
|
||||
_diff = diff.Diff(res_1, res_2, threshold)
|
||||
except exceptions.NotFoundException as e:
|
||||
print(six.text_type(e))
|
||||
def compare(self, *args, **kwargs):
|
||||
"""Deprecated."""
|
||||
# NOTE(amaretskiy): this command is deprecated in favor of
|
||||
# improved 'rally task results'
|
||||
print("This command is deprecated. Use 'rally task results' instead.")
|
||||
return 1
|
||||
|
||||
result = ""
|
||||
if output_json + output_html + output_csv > 1:
|
||||
print(_("Please specify only one output "
|
||||
"format: --json, --html or --csv."))
|
||||
return 1
|
||||
elif output_html:
|
||||
result = _diff.to_html()
|
||||
elif output_csv:
|
||||
result = _diff.to_csv()
|
||||
else:
|
||||
result = _diff.to_json()
|
||||
|
||||
if output_file:
|
||||
with open(output_file, "wb") as f:
|
||||
if output_csv:
|
||||
writer = csv.writer(f, dialect="excel")
|
||||
writer.writerows(result)
|
||||
else:
|
||||
f.write(result)
|
||||
else:
|
||||
print(result)
|
||||
|
||||
@cliutils.args("--uuid", type=str, dest="verification",
|
||||
required=False, help="UUID of a verification")
|
||||
def use(self, verification):
|
||||
|
||||
96
rally/ui/report.py
Normal file
96
rally/ui/report.py
Normal file
@ -0,0 +1,96 @@
|
||||
# Copyright 2016: Mirantis Inc.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Author: Oleksandr Savatieiev osavatieiev@mirantis.com
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
|
||||
from jinja2 import utils as jinja_utils
|
||||
|
||||
from rally.ui import utils
|
||||
|
||||
|
||||
class VerificationReport(object):
|
||||
|
||||
SKIP_RE = re.compile(
|
||||
"Skipped until Bug: ?(?P<bug_number>\d+) is resolved.")
|
||||
LP_BUG_LINK = "<a href='https://launchpad.net/bugs/{0}'>{0}</a>"
|
||||
|
||||
def __init__(self, verifications):
|
||||
self._runs = verifications
|
||||
self._uuids = list(verifications.keys())
|
||||
|
||||
# NOTE(amaretskiy): make aggregated list of all tests
|
||||
tests = {}
|
||||
for uuid, verification in self._runs.items():
|
||||
for name, test in verification["tests"].items():
|
||||
if name not in tests:
|
||||
# NOTE(amaretskiy): it is suitable to see resource id
|
||||
# at first place in the report
|
||||
tags = sorted(test["tags"], reverse=True,
|
||||
key=lambda tag: tag.startswith("id-"))
|
||||
tests[name] = {"name": name,
|
||||
"tags": tags,
|
||||
"by_verification": {},
|
||||
"has_details": False}
|
||||
tests[name]["by_verification"][uuid] = {
|
||||
"status": test["status"], "duration": test["duration"],
|
||||
"details": test["details"]}
|
||||
|
||||
if test["details"]:
|
||||
tests[name]["has_details"] = True
|
||||
match = self.SKIP_RE.match(test["details"])
|
||||
if match:
|
||||
href = self.LP_BUG_LINK.format(
|
||||
match.group("bug_number"))
|
||||
test["details"] = re.sub(
|
||||
match.group("bug_number"), href, test["details"])
|
||||
|
||||
test["details"] = jinja_utils.escape(test["details"])
|
||||
self._tests = list(tests.values())
|
||||
|
||||
def to_html(self):
|
||||
"""Make HTML report."""
|
||||
template = utils.get_template("verification/report.html")
|
||||
context = {"uuids": self._uuids, "verifications": self._runs,
|
||||
"tests": self._tests}
|
||||
return template.render(data=json.dumps(context), include_libs=False)
|
||||
|
||||
def to_json(self, indent=4):
|
||||
"""Make JSON report."""
|
||||
return json.dumps(self._tests, indent=indent)
|
||||
|
||||
def to_csv(self, **kwargs):
|
||||
"""Make CSV report."""
|
||||
header = ["test name", "tags", "has errors"]
|
||||
for uuid in self._uuids:
|
||||
header.extend(["%s status" % uuid, "%s duration" % uuid])
|
||||
rows = [header]
|
||||
for test in self._tests:
|
||||
row = [test["name"], " ".join(test["tags"])]
|
||||
for uuid in self._uuids:
|
||||
if uuid not in test["by_verification"]:
|
||||
row.extend([None, None])
|
||||
continue
|
||||
row.append(test["by_verification"][uuid]["status"])
|
||||
row.append(test["by_verification"][uuid]["duration"])
|
||||
rows.append(row)
|
||||
|
||||
with io.BytesIO() as stream:
|
||||
csv.writer(stream, **kwargs).writerows(rows)
|
||||
return stream.getvalue()
|
||||
@ -24,6 +24,7 @@
|
||||
table.compact td { padding:4px 8px }
|
||||
table.striped tr:nth-child(odd) td { background:#f9f9f9 }
|
||||
table.linked tbody tr:hover { background:#f9f9f9; cursor:pointer }
|
||||
.pointer { cursor:pointer }
|
||||
.rich, .rich td { font-weight:bold }
|
||||
.code { padding:10px; font-size:13px; color:#333; background:#f6f6f6; border:1px solid #e5e5e5; border-radius:4px }
|
||||
|
||||
@ -36,20 +37,20 @@
|
||||
.status-fail, .status-fail td { color:red }
|
||||
.capitalize { text-transform:capitalize }
|
||||
{% block css %}{% endblock %}
|
||||
.content-wrap { {% block css_content_wrap %}{% endblock %} margin:0 auto; padding:0 5px }
|
||||
.content-wrap { margin:0 auto; padding:0 5px; {% block css_content_wrap %}{% endblock %} }
|
||||
{% block media_queries %}{% endblock %}
|
||||
</style>
|
||||
</head>
|
||||
<body{% block body_attr %}{% endblock %}>
|
||||
|
||||
<div class="header">
|
||||
<div class="header" id="page-header">
|
||||
<div class="content-wrap">
|
||||
<a href="https://github.com/openstack/rally">Rally</a>
|
||||
<span>{% block header_text %}{% endblock %}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="content-wrap">
|
||||
<div class="content-wrap" id="page-content">
|
||||
{% block content %}{% endblock %}
|
||||
</div>
|
||||
|
||||
|
||||
@ -1,165 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>${heading["title"]}</title>
|
||||
<meta name="generator" content="${generator}">
|
||||
<meta charset="utf-8">
|
||||
<script type="text/javascript">
|
||||
var DOWN = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAGCAYAAAAVMmT4AAAAJUlEQVQYlWNgYGD4TwJmYCBFIYYGFhYWvArx2YAXEK0QWQMGAADd8SPpeGzm9QAAAABJRU5ErkJggg==";
|
||||
var NONE = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAGCAYAAAAVMmT4AAAADUlEQVQYlWNgGAUIAAABDgAB6WzgmwAAAABJRU5ErkJggg==";
|
||||
var UP = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAGCAYAAAAVMmT4AAAAK0lEQVQYlWNgwA7+4xDHqhCGiVaIVwNcAQsLC14N2EzEqoEYhf8ZGBj+AwCZbyPp8zIdEAAAAABJRU5ErkJggg==";
|
||||
|
||||
function sort_table(table_id, col, sort){
|
||||
var table = document.getElementById(table_id);
|
||||
var tbody = table.tBodies[0];
|
||||
var header_row = table.tHead.rows[0];
|
||||
render_header(col, sort, header_row);
|
||||
sort_results(tbody, col, sort);
|
||||
}
|
||||
|
||||
function render_header(col, sort, header_row){
|
||||
var h_cells = header_row.cells;
|
||||
for(i = 0; i < h_cells.length; i++){
|
||||
var cell = h_cells[i];
|
||||
var img = cell.firstElementChild;
|
||||
if (i == col){
|
||||
if (sort == 1){
|
||||
img.src = UP;
|
||||
}else{
|
||||
img.src = DOWN;
|
||||
}
|
||||
}else{ //spacer image
|
||||
img.src = NONE;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sort_results(tbody, col, sort) {
|
||||
var rows = tbody.rows, rlen = rows.length, arr = new Array(), i, j, cells, clen;
|
||||
// fill the array with values from the table
|
||||
for(i = 0; i < rlen; i++){
|
||||
cells = rows[i].cells;
|
||||
clen = cells.length;
|
||||
arr[i] = new Array();
|
||||
for(j = 0; j < clen; j++){
|
||||
arr[i][j] = cells[j].innerHTML;
|
||||
}
|
||||
}
|
||||
// sort the array by the specified column number (col) and order (sort)
|
||||
arr.sort(function(a, b){
|
||||
return (a[col] == b[col]) ? 0 : ((a[col] > b[col]) ? sort : -1*sort);
|
||||
});
|
||||
for(i = 0; i < rlen; i++){
|
||||
arr[i] = "<td>"+arr[i].join("</td><td>")+"</td>";
|
||||
}
|
||||
tbody.innerHTML = "<tr>"+arr.join("</tr><tr>")+"</tr>";
|
||||
}
|
||||
</script>
|
||||
<style type="text/css" media="screen">
|
||||
body {
|
||||
font-family: verdana, arial, helvetica, sans-serif;
|
||||
font-size: 80%;
|
||||
}
|
||||
|
||||
table {
|
||||
font-size: 100%; width: 100%;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 16pt;
|
||||
color: gray;
|
||||
}
|
||||
|
||||
.heading {
|
||||
margin-top: 0ex;
|
||||
margin-bottom: 1ex;
|
||||
}
|
||||
|
||||
.heading .attribute {
|
||||
margin-top: 1ex;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.heading .description {
|
||||
margin-top: 4ex;
|
||||
margin-bottom: 6ex;
|
||||
}
|
||||
|
||||
#results_table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
border: 1px solid #777;
|
||||
}
|
||||
|
||||
#header_row {
|
||||
font-weight: bold;
|
||||
color: white;
|
||||
background-color: #777;
|
||||
}
|
||||
|
||||
#results_table td {
|
||||
border: 1px solid #777;
|
||||
padding: 2px;
|
||||
}
|
||||
|
||||
.testcase { margin-left: 2em;}
|
||||
|
||||
img.updown{
|
||||
padding-left: 3px;
|
||||
padding-bottom: 2px;
|
||||
}
|
||||
|
||||
th:hover{
|
||||
cursor:pointer;
|
||||
}
|
||||
|
||||
.nowrap {white-space: nowrap;}
|
||||
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div class="heading">
|
||||
<h1>${heading["title"]}</h1>
|
||||
% for name, value in heading["parameters"]:
|
||||
<p class="attribute"><strong>${name}:</strong> ${value}</p>
|
||||
% endfor
|
||||
<p class="description">${heading["description"]}</p>
|
||||
</div>
|
||||
|
||||
<table id="results_table">
|
||||
<colgroup>
|
||||
<col align="left" />
|
||||
<col align="left" />
|
||||
<col align="left" />
|
||||
<col align="left" />
|
||||
<col align="left" />
|
||||
</colgroup>
|
||||
<thead>
|
||||
<tr id="header_row">
|
||||
<th class="nowrap" onclick="sort_table('results_table', 0, col1_sort); col1_sort *= -1; col2_sort = 1; col3_sort = 1; col4_sort = 1; col5_sort = 1;">Type<img class="updown" src=NONE /></th>
|
||||
<th class="nowrap" onclick="sort_table('results_table', 1, col2_sort); col2_sort *= -1; col1_sort = 1; col3_sort = 1; col4_sort = 1; col5_sort = 1;">Field<img class="updown" src=NONE /></th>
|
||||
<th class="nowrap" onclick="sort_table('results_table', 2, col3_sort); col3_sort *= -1; col1_sort = 1; col2_sort = 1; col4_sort = 1; col5_sort = 1;">Value 1<img class="updown" src=NONE /></th>
|
||||
<th class="nowrap" onclick="sort_table('results_table', 3, col4_sort); col4_sort *= -1; col1_sort = 1; col2_sort = 1; col3_sort = 1; col5_sort = 1;">Value 2<img class="updown" src=NONE /></th>
|
||||
<th onclick="sort_table('results_table', 4, col5_sort); col5_sort *= -1; col1_sort = 1; col2_sort = 1; col3_sort = 1; col4_sort = 1;">Test Name<img class="updown" src=NONE /></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="results">
|
||||
% for diff in results:
|
||||
<tr class="">
|
||||
<td class="type">${diff.get("type")}</td>
|
||||
<td class="field">${diff.get("field", "")}</td>
|
||||
<td class="val">${diff.get("val1", "")}</td>
|
||||
<td class="val">${diff.get("val2", "")}</td>
|
||||
<td class="testname">${diff.get("test_name")}</td>
|
||||
</tr>
|
||||
% endfor
|
||||
</table>
|
||||
<script type="text/javascript">
|
||||
var col1_sort = 1, col2_sort = 1, col3_sort = 1; col4_sort = 1; col5_sort = 1;
|
||||
sort_table("results_table", 4, col5_sort);
|
||||
col5_sort *= -1;
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
321
rally/ui/templates/verification/report.html
Normal file
321
rally/ui/templates/verification/report.html
Normal file
@ -0,0 +1,321 @@
|
||||
{% extends "/base.html" %}
|
||||
|
||||
{% block html_attr %} ng-app="App" ng-controller="Controller" id="page-html"{% endblock %}
|
||||
|
||||
{% block title_text %}{% raw %}{{title}}{% endraw %}{% endblock %}
|
||||
|
||||
{% block libs %}
|
||||
{% if include_libs %}
|
||||
<script type="text/javascript">
|
||||
{{ include_raw_file("/libs/angular.1.3.3.min.js") }}
|
||||
</script>
|
||||
{% else %}
|
||||
<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/angularjs/1.3.3/angular.min.js"></script>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
|
||||
{% block js_before %}
|
||||
"use strict";
|
||||
{{ include_raw_file("/task/directive_widget.js") }}
|
||||
var controllerFunction = function($scope, $location) {
|
||||
$scope.data = {{ data }};
|
||||
|
||||
/* Calculate columns width in percent */
|
||||
var td_ctr_width = 4;
|
||||
var td_result_width = Math.round(1 / ($scope.data.uuids.length+3) * 100);
|
||||
|
||||
$scope.td_width_ = {
|
||||
counter: td_ctr_width,
|
||||
test_name: (100 - td_ctr_width - (td_result_width * $scope.data.uuids.length)),
|
||||
test_result: td_result_width
|
||||
}
|
||||
|
||||
$scope.td_width = (function(vers_num) {
|
||||
var uuid_w = Math.round(1 / (vers_num+3) * 100);
|
||||
return {test: 100 - (uuid_w * vers_num),
|
||||
uuid: uuid_w}
|
||||
})($scope.data.uuids.length)
|
||||
|
||||
var bitmask = {"success": 1,
|
||||
"x-fail": 2,
|
||||
"skip": 4,
|
||||
"ux-ok": 8,
|
||||
"fail": 16};
|
||||
|
||||
for (var i in $scope.data.tests) {
|
||||
var t = $scope.data.tests[i];
|
||||
var bits = 0;
|
||||
for (var uuid in t.by_verification) {
|
||||
var status = t.by_verification[uuid].status;
|
||||
if (status in bitmask) {
|
||||
bits |= bitmask[status]
|
||||
}
|
||||
$scope.data.tests[i].by_verification[uuid].show_duration = (
|
||||
t.by_verification[uuid].duration > 0.0001)
|
||||
}
|
||||
$scope.data.tests[i].filter = bits;
|
||||
}
|
||||
|
||||
$scope.set_filter = function(status) {
|
||||
if (status in $scope.state) {
|
||||
$scope.state[status] = !$scope.state[status];
|
||||
$scope.filter_bits ^= bitmask[status]
|
||||
}
|
||||
}
|
||||
|
||||
$scope.state = {"success": true,
|
||||
"x-fail": true,
|
||||
"skip": true,
|
||||
"ux-ok": true,
|
||||
"fail": true};
|
||||
|
||||
$scope.filter_by_status = function(test, index, arr) {
|
||||
return test.filter & $scope.filter_bits
|
||||
}
|
||||
|
||||
$scope.filter_bits = (function(filter){
|
||||
var bits = 0;
|
||||
for (var status in $scope.state){
|
||||
if ($scope.state[status]) { bits ^= bitmask[status] }
|
||||
}
|
||||
return bits
|
||||
})();
|
||||
|
||||
$scope.toggle_filters_flag = true;
|
||||
$scope.toggle_filters = function() {
|
||||
if ($scope.toggle_filters_flag) {
|
||||
$scope.toggle_filters_flag = false;
|
||||
$scope.state = {"success": false,
|
||||
"x-fail": false,
|
||||
"skip": false,
|
||||
"ux-ok": false,
|
||||
"fail": false};
|
||||
$scope.filter_bits = 0
|
||||
} else {
|
||||
$scope.toggle_filters_flag = true
|
||||
$scope.state = {"success": true,
|
||||
"x-fail": true,
|
||||
"skip": true,
|
||||
"ux-ok": true,
|
||||
"fail": true};
|
||||
$scope.filter_bits = 31
|
||||
}
|
||||
}
|
||||
|
||||
var title = "verification result";
|
||||
|
||||
if ($scope.data.uuids.length > 1) {
|
||||
title = "verifications results"
|
||||
}
|
||||
|
||||
$scope.title = title;
|
||||
|
||||
$scope.srt_dir = false;
|
||||
|
||||
$scope.get_tests_count = function() {
|
||||
var ctr = 0;
|
||||
for (var i in $scope.data.tests) {
|
||||
if ($scope.data.tests[i].filter & $scope.filter_bits) {
|
||||
ctr++
|
||||
}
|
||||
}
|
||||
return ctr
|
||||
}
|
||||
|
||||
var title = angular.element(document.getElementById("page-header"));
|
||||
var header = angular.element(document.getElementById("content-header"));
|
||||
var tests = angular.element(document.getElementById("tests"));
|
||||
var sync_positions = function() {
|
||||
var title_h = title[0].offsetHeight;
|
||||
var header_h = header[0].offsetHeight;
|
||||
header.css({top:title_h+"px"})
|
||||
tests.css({"margin-top": (title_h+header_h)+"px"});
|
||||
}
|
||||
|
||||
/* Make page head sticky */
|
||||
window.onload = function() {
|
||||
title.css({position:"fixed", top:0, width:"100%"});
|
||||
header.css({position:"fixed", width:"100%", background:"#fff"});
|
||||
|
||||
sync_positions();
|
||||
window.onresize = sync_positions;
|
||||
|
||||
var gotop = document.getElementById("button-gotop");
|
||||
gotop.onclick = function () { scrollTo(0, 0) };
|
||||
window.onscroll = function() {
|
||||
if (window.scrollY > 50) {
|
||||
gotop.style.display = "block";
|
||||
} else {
|
||||
gotop.style.display = "none";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$scope.toggle_header = (function(e) {
|
||||
return function() {
|
||||
e.style.display = (e.style.display === "none") ? "table" : "none";
|
||||
sync_positions()
|
||||
}
|
||||
})(document.getElementById("verifications"))
|
||||
|
||||
};
|
||||
|
||||
if (typeof angular === "object") {
|
||||
angular.module("App", [])
|
||||
.controller("Controller", ["$scope", "$location", controllerFunction])
|
||||
.directive("widget", widgetDirective)
|
||||
}
|
||||
{% endblock %}
|
||||
|
||||
{% block css %}
|
||||
div.header {margin:0 !important}
|
||||
div.header .content-wrap { padding-left:10px }
|
||||
.status.status-success { background: #cfc; color: #333 }
|
||||
.status.status-ux-ok { background: #ffd7af; color: #333 }
|
||||
.status.status-fail { background: #fbb; color: #333 }
|
||||
.status.status-x-fail { background: #ccf5ff; color: #333 }
|
||||
.status.status-skip { background: #ffb; color: #333 }
|
||||
.status.checkbox { font-size:18px; text-align:center; cursor:pointer; padding:0 }
|
||||
.column { display:block; float:left; padding:4px 0 4px 8px; box-sizing:border-box;
|
||||
background:#fff; font-size:12px; font-weight:bold;
|
||||
border:#ccc solid; border-width:0 0 1px }
|
||||
.button { margin:0 5px; padding:0 8px 1px; background:#47a; color:#fff; cursor:pointer;
|
||||
border:1px #036 solid; border-radius:11px; font-size:12px; font-weight:normal;
|
||||
line-height:12px; opacity:.8}
|
||||
.button:hover { opacity:1 }
|
||||
#button-gotop { padding:3px 10px 5px; text-align:center; cursor:pointer;
|
||||
background:#fff; color:#036; line-height:14px; font-size:14px;
|
||||
position:fixed; bottom:0; right:10px;
|
||||
border:#ccc solid; border-width:1px 1px 0; border-radius:15px 15px 0 0}
|
||||
{% endblock %}
|
||||
|
||||
{% block css_content_wrap %}width:100%; padding:0{% endblock %}
|
||||
|
||||
{% block body_attr %} id="page-body" style="position:relative"{% endblock %}
|
||||
|
||||
{% block header_text %}{% raw %}{{title}}{% endraw %}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
|
||||
{% raw %}
|
||||
<h3 ng-hide="true" style="padding-left:10px">processing ...</h3>
|
||||
|
||||
<div id="content-header" ng-cloak>
|
||||
<table class="compact" id="verifications"
|
||||
style="border:#fff solid; border-width:2px 0 15px; margin:0">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>UUID
|
||||
<th>Set name
|
||||
<th>Status
|
||||
<th>Started at
|
||||
<th>Duration, s
|
||||
<th>Total tests
|
||||
<th style="width:9%">success
|
||||
<th style="width:9%">expected failures
|
||||
<th style="width:9%">skipped
|
||||
<th style="width:9%">unexpected success
|
||||
<th style="width:9%">failures
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr ng-repeat="uuid in data.uuids">
|
||||
<td>{{uuid}}
|
||||
<td>{{data.verifications[uuid].set_name}}
|
||||
<td>{{data.verifications[uuid].status}}
|
||||
<td>{{data.verifications[uuid].started_at}}
|
||||
<td>{{data.verifications[uuid].duration}}
|
||||
<td>{{data.verifications[uuid].total}}
|
||||
<td class="status status-success">{{data.verifications[uuid].success}}
|
||||
<td class="status status-x-fail">{{data.verifications[uuid].expected_failures}}
|
||||
<td class="status status-skip">{{data.verifications[uuid].skipped}}
|
||||
<td class="status status-ux-ok">{{data.verifications[uuid].unexpected_success}}
|
||||
<td class="status status-fail">{{data.verifications[uuid].failures}}
|
||||
</tr>
|
||||
<tr>
|
||||
<td colspan="6" style="text-align:right; font-weight:bold">
|
||||
<span ng-click="toggle_filters()" class="button" style="margin-right:10px">
|
||||
toggle all filters
|
||||
</span>
|
||||
Filter tests by status:
|
||||
<td class="checkbox status status-success" ng-click="set_filter('success')">
|
||||
<span ng-hide="state.success">☐</span>
|
||||
<span ng-show="state.success">☑</span>
|
||||
<td class="checkbox status status-x-fail" ng-click="set_filter('x-fail')">
|
||||
<span ng-hide="state['x-fail']">☐</span>
|
||||
<span ng-show="state['x-fail']">☑</span>
|
||||
<td class="checkbox status status-skip" ng-click="set_filter('skip')">
|
||||
<span ng-hide="state.skip">☐</span>
|
||||
<span ng-show="state.skip">☑</span>
|
||||
<td class="checkbox status status-ux-ok" ng-click="set_filter('ux-ok')">
|
||||
<span ng-hide="state['ux-ok']">☐</span>
|
||||
<span ng-show="state['ux-ok']">☑</span>
|
||||
<td class="checkbox status status-fail" ng-click="set_filter('fail')">
|
||||
<span ng-hide="state.fail">☐</span>
|
||||
<span ng-show="state.fail">☑</span>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<div style="text-align:left; padding:6px 3px; background:#fff">
|
||||
<span class="button" ng-click="show_tags=!show_tags">
|
||||
toggle tags
|
||||
</span>
|
||||
<span class="button" ng-click="toggle_header()">
|
||||
toggle header
|
||||
</span>
|
||||
</div>
|
||||
<div style="clear:both"></div>
|
||||
|
||||
<div style="width:{{td_width.test}}%" class="column">
|
||||
<span ng-click="srt_dir=!srt_dir" class="pointer">
|
||||
Test name
|
||||
<span style="color:#777">(shown {{get_tests_count()}})</span>
|
||||
<span style="color:orange">
|
||||
<span ng-hide="srt_dir">▾</span>
|
||||
<span ng-show="srt_dir">▴</span>
|
||||
</span>
|
||||
</span>
|
||||
</div>
|
||||
<div ng-repeat="uuid in data.uuids"
|
||||
class="column"
|
||||
style="width:{{td_width.uuid}}%; white-space:nowrap; overflow:hidden; text-overflow:ellipsis">
|
||||
{{uuid}}
|
||||
</div>
|
||||
<div style="clear:both"></div>
|
||||
</div>
|
||||
|
||||
<table class="compact" id="tests" style="margin:0; width:100%" ng-cloak>
|
||||
<tbody ng-repeat="t in data.tests | orderBy:'name':srt_dir track by $index" ng-show="filter_by_status(t)">
|
||||
<tr ng-click="t.expanded=!t.expanded" ng-class="{pointer:t.has_details}">
|
||||
<td style="width:{{td_width.test}}%; word-break:break-all">
|
||||
{{t.name}}
|
||||
<div ng-show="show_tags" style="font-size:12px; color:#999; word-break:normal">
|
||||
<span ng-repeat="tag in t.tags"> {{tag}}</span>
|
||||
</div>
|
||||
<td ng-repeat="uuid in data.uuids"
|
||||
class="status status-{{t.by_verification[uuid].status}}"
|
||||
style="width:{{td_width.uuid}}%">
|
||||
<div ng-if="t.by_verification[uuid]">
|
||||
{{t.by_verification[uuid].status}}
|
||||
<span ng-if="t.by_verification[uuid].show_duration">{{t.by_verification[uuid].duration}}</span>
|
||||
</div>
|
||||
<div ng-if="!t.by_verification[uuid]" style="color:#999">
|
||||
–
|
||||
</div>
|
||||
</tr>
|
||||
<tr ng-if="t.has_details" ng-show="t.expanded" style="width:100%">
|
||||
<td colspan="{{3+data.uuids.length}}" style="padding:0">
|
||||
<div ng-repeat="uuid in data.uuids" ng-if="t.by_verification[uuid].details"
|
||||
class="status status-{{t.by_verification[uuid].status}}"
|
||||
style="padding:5px">
|
||||
<div style="font-weight:bold; color:#333">{{uuid}}</div>
|
||||
<pre style="text-overflow:hidden">{{t.by_verification[uuid].details}}</pre>
|
||||
</div>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<span id="button-gotop" style="display:none">go top</span>
|
||||
{% endraw %}
|
||||
|
||||
{% endblock %}
|
||||
@ -1,129 +0,0 @@
|
||||
## -*- coding: utf-8 -*-
|
||||
<%inherit file="/base.mako"/>
|
||||
|
||||
<%block name="title_text">Tempest report</%block>
|
||||
|
||||
<%block name="libs">
|
||||
<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
|
||||
</%block>
|
||||
|
||||
<%block name="css">
|
||||
.test-details-row { display:none }
|
||||
.test-details { font-family:monospace; white-space:pre; overflow:auto }
|
||||
.test-expandable { cursor:pointer }
|
||||
.test-expandable:hover { background:#f3f3f3 }
|
||||
|
||||
.nav { margin: 15px 0 }
|
||||
.nav span { padding:1px 15px; margin:0 2px 0 0; cursor:pointer; background:#f3f3f3;
|
||||
color: black; font-size:12px; border:2px #ddd solid; border-radius:10px }
|
||||
.nav span.active { background:#cfe3ff; border-color:#ace; color:#369 }
|
||||
|
||||
table td { padding:4px 8px; word-wrap:break-word; word-break:break-all }
|
||||
table.stat { width:auto; margin:0 0 15px }
|
||||
td.not_break_column {word-break:keep-all}
|
||||
|
||||
.status-success, .status-success td { color:green }
|
||||
.status-uxsuccess, .status-uxsuccess td { color:orange }
|
||||
.status-xfail, .status-xfail td { color:#CCCC00}
|
||||
</%block>
|
||||
|
||||
<%block name="css_content_wrap">
|
||||
margin:0 auto; padding:0 5px
|
||||
</%block>
|
||||
|
||||
<%block name="media_queries">
|
||||
@media only screen and (min-width: 300px) { .content-wrap { width:370px } .test-details { width:360px } }
|
||||
@media only screen and (min-width: 500px) { .content-wrap { width:470px } .test-details { width:460px } }
|
||||
@media only screen and (min-width: 600px) { .content-wrap { width:570px } .test-details { width:560px } }
|
||||
@media only screen and (min-width: 700px) { .content-wrap { width:670px } .test-details { width:660px } }
|
||||
@media only screen and (min-width: 800px) { .content-wrap { width:770px } .test-details { width:760px } }
|
||||
@media only screen and (min-width: 900px) { .content-wrap { width:870px } .test-details { width:860px } }
|
||||
@media only screen and (min-width: 1000px) { .content-wrap { width:970px } .test-details { width:960px } }
|
||||
@media only screen and (min-width: 1200px) { .content-wrap { width:auto } .test-details { width:94% } }
|
||||
</%block>
|
||||
|
||||
<%block name="header_text">Tempest Report</%block>
|
||||
|
||||
<%block name="content">
|
||||
<p id="page-error" class="notify-error" style="display:none">Failed to load jQuery</p>
|
||||
|
||||
<table class="stat">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Total
|
||||
<th>Total Time
|
||||
<th>Success
|
||||
<th>Fails
|
||||
<th>Unexpected Success
|
||||
<th>Expected Fails
|
||||
<th>Skipped
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>${report['total']}
|
||||
<td>${report['time']}
|
||||
<td>${report['success']}
|
||||
<td>${report['failures']}
|
||||
<td>${report['unexpected_success']}
|
||||
<td>${report['expected_failures']}
|
||||
<td>${report['skipped']}
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<div class="nav">
|
||||
<span data-navselector=".test-row">all</span>
|
||||
<span data-navselector=".status-success">success</span>
|
||||
<span data-navselector=".status-fail">failed</span>
|
||||
<span data-navselector=".status-uxsuccess">uxsuccess</span>
|
||||
<span data-navselector=".status-xfail">xfailed</span>
|
||||
<span data-navselector=".status-skip">skipped</span>
|
||||
</div>
|
||||
|
||||
<table id="tests">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Status
|
||||
<th>Time
|
||||
<th colspan="5">Test Case
|
||||
<tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
% for test in report['tests']:
|
||||
<tr id="${test['id']}" class="test-row status-${test['status']}">
|
||||
<td class="not_break_column">${test['status']}
|
||||
<td class="not_break_column">${test['time']}
|
||||
<td colspan="5">${test['name']}
|
||||
</tr>
|
||||
% if 'output' in test:
|
||||
<tr class="test-details-row">
|
||||
<td colspan="6"><div class="test-details">${test['output'] | n}</div>
|
||||
</tr>
|
||||
% endif
|
||||
% endfor
|
||||
</tbody>
|
||||
</table>
|
||||
</%block>
|
||||
|
||||
<%block name="js_after">
|
||||
if (typeof $ === "undefined") {
|
||||
/* If jQuery loading has failed */
|
||||
document.getElementById("page-error").style.display = "block"
|
||||
} else {
|
||||
$(function(){
|
||||
$(".test-details-row")
|
||||
.prev()
|
||||
.addClass("test-expandable")
|
||||
.click( function(){ $(this).next().toggle() });
|
||||
|
||||
(function($navs) {
|
||||
$navs.click(function(){
|
||||
var $this = $(this);
|
||||
$navs.removeClass("active").filter($this).addClass("active");
|
||||
$("#tests tbody tr").hide().filter($this.attr("data-navselector")).show();
|
||||
}).first().click()
|
||||
}($(".nav [data-navselector]")));
|
||||
})
|
||||
}
|
||||
</%block>
|
||||
@ -1,35 +0,0 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""Output verification comparison results in html."""
|
||||
|
||||
from rally.ui import utils as ui_utils
|
||||
|
||||
__description__ = "List differences between two verification runs"
|
||||
__title__ = "Verification Comparison"
|
||||
__version__ = "0.1"
|
||||
|
||||
|
||||
def create_report(results):
|
||||
template_kw = {
|
||||
"heading": {
|
||||
"title": __title__,
|
||||
"description": __description__,
|
||||
"parameters": [("Difference Count", len(results))]
|
||||
},
|
||||
"generator": "compare2html %s" % __version__,
|
||||
"results": results
|
||||
}
|
||||
|
||||
template = ui_utils.get_template("verification/compare.mako")
|
||||
output = template.render(**template_kw)
|
||||
|
||||
return output.encode("utf8")
|
||||
@ -1,105 +0,0 @@
|
||||
# Copyright 2014 Dell Inc.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import json
|
||||
|
||||
from rally.verification.tempest import compare2html
|
||||
|
||||
|
||||
class Diff(object):
|
||||
|
||||
def __init__(self, test_cases1, test_cases2, threshold):
|
||||
"""Compare two verification results.
|
||||
|
||||
Compares two verification results and emits
|
||||
desired output, csv, html, json or pprint.
|
||||
|
||||
:param test_cases1: older verification json
|
||||
:param test_cases2: newer verification json
|
||||
:param threshold: test time difference percentage threshold
|
||||
|
||||
"""
|
||||
self.threshold = threshold
|
||||
self.diffs = self._compare(test_cases1, test_cases2)
|
||||
|
||||
def _compare(self, tc1, tc2):
|
||||
"""Compare two verification results.
|
||||
|
||||
:param tc1: first verification test cases json
|
||||
:param tc2: second verification test cases json
|
||||
|
||||
Typical test case json schema:
|
||||
"test_case_key": {
|
||||
"traceback": "", # exists only for "fail" status
|
||||
"reason": "", # exists only for "skip" status
|
||||
"name": "",
|
||||
"status": "",
|
||||
"time": 0.0
|
||||
}
|
||||
"""
|
||||
diffs = []
|
||||
names1 = set(tc1.keys())
|
||||
names2 = set(tc2.keys())
|
||||
|
||||
common_tests = list(names1.intersection(names2))
|
||||
removed_tests = list(names1.difference(common_tests))
|
||||
new_tests = list(names2.difference(common_tests))
|
||||
|
||||
for name in removed_tests:
|
||||
diffs.append({"type": "removed_test", "test_name": name})
|
||||
for name in new_tests:
|
||||
diffs.append({"type": "new_test", "test_name": name})
|
||||
for name in common_tests:
|
||||
diffs.extend(self._diff_values(name, tc1[name], tc2[name]))
|
||||
|
||||
return diffs
|
||||
|
||||
def _diff_values(self, name, result1, result2):
|
||||
fields = ["status", "time", "traceback", "reason"]
|
||||
diffs = []
|
||||
for field in fields:
|
||||
val1 = result1.get(field, 0)
|
||||
val2 = result2.get(field, 0)
|
||||
if val1 != val2:
|
||||
if field == "time":
|
||||
max_ = max(float(val1), float(val2))
|
||||
min_ = min(float(val1), float(val2))
|
||||
time_threshold = ((max_ - min_) / (min_ or 1)) * 100
|
||||
if time_threshold < self.threshold:
|
||||
continue
|
||||
|
||||
diffs.append({
|
||||
"field": field,
|
||||
"type": "value_changed",
|
||||
"test_name": name,
|
||||
"val1": val1,
|
||||
"val2": val2
|
||||
})
|
||||
return diffs
|
||||
|
||||
def to_csv(self):
|
||||
rows = (("Type", "Field", "Value 1", "Value 2", "Test Name"),)
|
||||
for res in self.diffs:
|
||||
row = (res.get("type"), res.get("field", ""),
|
||||
res.get("val1", ""), res.get("val2", ""),
|
||||
res.get("test_name"))
|
||||
rows = rows + (row,)
|
||||
return rows
|
||||
|
||||
def to_json(self):
|
||||
return json.dumps(self.diffs, sort_keys=True, indent=4)
|
||||
|
||||
def to_html(self):
|
||||
return compare2html.create_report(self.diffs)
|
||||
@ -1,61 +0,0 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import datetime as dt
|
||||
import re
|
||||
|
||||
from jinja2 import utils
|
||||
|
||||
from rally.ui import utils as ui_utils
|
||||
|
||||
SKIP_RE = re.compile("Skipped until Bug: ?(?P<bug_number>\d+) is resolved.")
|
||||
LAUNCHPAD_BUG_LINK = "<a href='https://launchpad.net/bugs/{0}'>{0}</a>"
|
||||
|
||||
|
||||
def generate_report(results):
|
||||
"""Generates HTML report from test results in JSON format."""
|
||||
tests = []
|
||||
for i, name in enumerate(sorted(results["test_cases"])):
|
||||
test = results["test_cases"][name]
|
||||
output = ""
|
||||
if "reason" in test:
|
||||
output += "Reason:\n "
|
||||
matcher = SKIP_RE.match(test["reason"])
|
||||
if matcher:
|
||||
href = LAUNCHPAD_BUG_LINK.format(matcher.group("bug_number"))
|
||||
output += re.sub(matcher.group("bug_number"), href,
|
||||
test["reason"])
|
||||
else:
|
||||
output += utils.escape(test["reason"])
|
||||
if "traceback" in test:
|
||||
if output:
|
||||
output += "\n\n"
|
||||
output += utils.escape(test["traceback"])
|
||||
|
||||
tests.append({"id": i,
|
||||
"time": test["time"],
|
||||
"name": name,
|
||||
"output": output,
|
||||
"status": test["status"]})
|
||||
|
||||
template = ui_utils.get_template("verification/report.mako")
|
||||
return template.render(report={
|
||||
"tests": tests,
|
||||
"total": results["tests"],
|
||||
"time": "{0} ({1} s)".format(
|
||||
dt.timedelta(seconds=round(
|
||||
float(results["time"]))), results["time"]),
|
||||
"success": results["success"],
|
||||
"failures": results["failures"],
|
||||
"skipped": results["skipped"],
|
||||
"expected_failures": results["expected_failures"],
|
||||
"unexpected_success": results["unexpected_success"]})
|
||||
@ -140,7 +140,7 @@ def do_compare(uuid_1, uuid_2):
|
||||
"""Compare and save results in different formats."""
|
||||
results = {}
|
||||
for output_format in ("csv", "html", "json"):
|
||||
cmd = "verify compare --uuid-1 %(uuid-1)s --uuid-2 %(uuid-2)s" % {
|
||||
cmd = "verify results --uuid %(uuid-1)s %(uuid-2)s" % {
|
||||
"uuid-1": uuid_1,
|
||||
"uuid-2": uuid_2
|
||||
}
|
||||
|
||||
@ -17,6 +17,7 @@ import datetime as dt
|
||||
import os.path
|
||||
import tempfile
|
||||
|
||||
import ddt
|
||||
import mock
|
||||
|
||||
from rally.cli.commands import verify
|
||||
@ -25,6 +26,7 @@ from rally import exceptions
|
||||
from tests.unit import test
|
||||
|
||||
|
||||
@ddt.ddt
|
||||
class VerifyCommandsTestCase(test.TestCase):
|
||||
def setUp(self):
|
||||
super(VerifyCommandsTestCase, self).setUp()
|
||||
@ -266,16 +268,42 @@ class VerifyCommandsTestCase(test.TestCase):
|
||||
mock_print_list.call_args_list)
|
||||
mock_verification.get.assert_called_once_with(verification_id)
|
||||
|
||||
@ddt.data({"uuids": ["foo_uuid"], "expected_uuid": "foo_uuid"},
|
||||
{"uuids": None, "expected_uuid": "DFLT_UUID"})
|
||||
@ddt.unpack
|
||||
@mock.patch("rally.cli.commands.verify.envutils")
|
||||
@mock.patch("rally.api.Verification")
|
||||
@mock.patch("json.dumps")
|
||||
def test_results(self, mock_json_dumps, mock_verification):
|
||||
def test_results(self, mock_json_dumps, mock_verification, mock_envutils,
|
||||
uuids, expected_uuid):
|
||||
mock_envutils.get_global.return_value = "DFLT_UUID"
|
||||
mock_verification.get.return_value.get_results.return_value = {}
|
||||
verification_uuid = "a0231bdf-6a4e-4daf-8ab1-ae076f75f070"
|
||||
self.verify.results(verification_uuid, output_html=False,
|
||||
self.verify.results(uuids, output_html=False,
|
||||
output_json=True)
|
||||
|
||||
mock_verification.get.assert_called_once_with(verification_uuid)
|
||||
mock_json_dumps.assert_called_once_with({}, sort_keys=True, indent=4)
|
||||
mock_verification.get.assert_called_once_with(expected_uuid)
|
||||
mock_json_dumps.assert_called_once_with([], indent=4)
|
||||
if uuids:
|
||||
self.assertFalse(mock_envutils.get_global.called)
|
||||
else:
|
||||
mock_envutils.get_global.assert_called_once_with(
|
||||
mock_envutils.ENV_VERIFICATION)
|
||||
|
||||
@mock.patch("rally.api.Verification")
|
||||
def test_results_many_formats_given(self, mock_verification):
|
||||
mock_verification.get.return_value.get_results.return_value = {}
|
||||
self.assertEqual(1, self.verify.results(
|
||||
["foo_uuid"], output_html=True, output_json=True))
|
||||
mock_verification.get.assert_called_once_with("foo_uuid")
|
||||
|
||||
@mock.patch("rally.api.Verification")
|
||||
def test_results_duplicated_test(self, mock_verification):
|
||||
ver_res = {"test_cases": {
|
||||
"a": {"name": "foo", "tags": [], "status": "success", "time": 4},
|
||||
"b": {"name": "foo", "tags": [], "status": "success", "time": 2}}}
|
||||
mock_verification.get.return_value.get_results.return_value = ver_res
|
||||
self.assertRaises(exceptions.RallyException, self.verify.results,
|
||||
["uuid"], output_html=False, output_json=True)
|
||||
|
||||
@mock.patch("rally.api.Verification.get")
|
||||
def test_results_verification_not_found(
|
||||
@ -284,8 +312,7 @@ class VerifyCommandsTestCase(test.TestCase):
|
||||
mock_verification_get.side_effect = (
|
||||
exceptions.NotFoundException()
|
||||
)
|
||||
self.assertEqual(self.verify.results(verification_uuid,
|
||||
output_html=False,
|
||||
self.assertEqual(self.verify.results([verification_uuid],
|
||||
output_json=True), 1)
|
||||
|
||||
mock_verification_get.assert_called_once_with(verification_uuid)
|
||||
@ -298,128 +325,88 @@ class VerifyCommandsTestCase(test.TestCase):
|
||||
mock_verification.get.return_value.get_results.return_value = {}
|
||||
mock_open.side_effect = mock.mock_open()
|
||||
verification_uuid = "94615cd4-ff45-4123-86bd-4b0741541d09"
|
||||
self.verify.results(verification_uuid, output_file="results",
|
||||
self.verify.results([verification_uuid], output_file="results",
|
||||
output_html=False, output_json=True)
|
||||
|
||||
mock_verification.get.assert_called_once_with(verification_uuid)
|
||||
mock_open.assert_called_once_with("results", "wb")
|
||||
mock_open.side_effect().write.assert_called_once_with("{}")
|
||||
mock_open.side_effect().write.assert_called_once_with("[]")
|
||||
|
||||
@mock.patch("rally.cli.commands.verify.report.VerificationReport")
|
||||
@mock.patch("rally.cli.commands.verify.open",
|
||||
side_effect=mock.mock_open(), create=True)
|
||||
@mock.patch("rally.api.Verification")
|
||||
@mock.patch("rally.verification.tempest.json2html.generate_report")
|
||||
@mock.patch("rally.cli.commands.verify.api.Verification")
|
||||
def test_results_with_output_html_and_output_file(
|
||||
self, mock_generate_report, mock_verification, mock_open):
|
||||
|
||||
self, mock_verification, mock_open, mock_verification_report):
|
||||
verification_uuid = "7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa"
|
||||
self.verify.results(verification_uuid, output_html=True,
|
||||
output_json=False, output_file="results")
|
||||
mock_vr = mock.Mock()
|
||||
mock_verification_report.return_value = mock_vr
|
||||
mock_strftime_created = mock.Mock(return_value="ts_created")
|
||||
mock_strftime_updated = mock.Mock(return_value="ts_updated")
|
||||
mock_test_cases = mock.Mock()
|
||||
mock_test_cases.values.return_value = [
|
||||
{"name": "foo_name", "status": "success",
|
||||
"time": 10.0, "tags": ["foo-tag", "id-foo"]},
|
||||
{"name": "bar_name", "status": "success",
|
||||
"time": 30.1, "tags": ["bar-tag", "id-bar"]},
|
||||
{"name": "spam_name", "status": "skip",
|
||||
"time": 0, "tags": ["id-spam", "spam-tag"]},
|
||||
{"name": "quiz_name", "status": "fail", "time": 0,
|
||||
"tags": ["quiz-tag", "id-quiz"], "traceback": " Quiz error "}]
|
||||
results = {
|
||||
"test_cases": mock_test_cases,
|
||||
"time": 42.1,
|
||||
"tests": 4,
|
||||
"skipped": 1,
|
||||
"success": 2,
|
||||
"expected_failures": 0,
|
||||
"unexpected_success": 0,
|
||||
"failures": 1}
|
||||
mock_verification.get.return_value.db_object = {
|
||||
"uuid": verification_uuid,
|
||||
"created_at": mock.Mock(strftime=mock_strftime_created),
|
||||
"updated_at": mock.Mock(strftime=mock_strftime_updated),
|
||||
"status": "foo_status",
|
||||
"set_name": "foo_set"}
|
||||
mock_verification.get.return_value.get_results.return_value = results
|
||||
expected = {
|
||||
"status": "foo_status",
|
||||
"tests": {
|
||||
"foo_name": {"status": "success", "duration": 10.0,
|
||||
"details": None, "tags": ["foo-tag", "id-foo"]},
|
||||
"spam_name": {"status": "skip", "duration": 0, "details": None,
|
||||
"tags": ["id-spam", "spam-tag"]},
|
||||
"quiz_name": {"status": "fail", "duration": 0,
|
||||
"details": "Quiz error",
|
||||
"tags": ["quiz-tag", "id-quiz"]},
|
||||
"bar_name": {"status": "success", "duration": 30.1,
|
||||
"details": None, "tags": ["bar-tag", "id-bar"]}},
|
||||
"skipped": 1, "finished_at": "ts_updated", "duration": 42.1,
|
||||
"started_at": "ts_created", "set_name": "foo_set", "total": 4,
|
||||
"success": 2, "expected_failures": 0, "failures": 1,
|
||||
"unexpected_success": 0}
|
||||
|
||||
self.verify.results([verification_uuid], output_html=True,
|
||||
output_file="results")
|
||||
|
||||
mock_verification.get.assert_called_once_with(verification_uuid)
|
||||
mock_generate_report.assert_called_once_with(
|
||||
mock_verification.get.return_value.get_results.return_value)
|
||||
mock_verification_report.assert_called_once_with(
|
||||
{verification_uuid: expected})
|
||||
mock_open.assert_called_once_with("results", "wb")
|
||||
mock_open.side_effect().write.assert_called_once_with(
|
||||
mock_generate_report.return_value)
|
||||
mock_vr.to_html.return_value)
|
||||
|
||||
@mock.patch("rally.api.Verification")
|
||||
@mock.patch("json.dumps")
|
||||
def test_compare(self, mock_json_dumps, mock_verification):
|
||||
mock_verification.get.return_value.get_results.return_value = {
|
||||
"test_cases": {}}
|
||||
uuid1 = "8eda1b10-c8a4-4316-9603-8468ff1d1560"
|
||||
uuid2 = "f6ef0a98-1b18-452f-a6a7-922555c2e326"
|
||||
self.verify.compare(uuid1, uuid2, output_csv=False, output_html=False,
|
||||
output_json=True)
|
||||
@mock.patch("rally.cli.commands.verify.envutils")
|
||||
def test_results_no_uuid_given(self, mock_envutils):
|
||||
mock_envutils.get_global.return_value = None
|
||||
self.assertRaises(exceptions.InvalidArgumentsException,
|
||||
self.verify.results, None,
|
||||
output_html=True, output_file="results")
|
||||
mock_envutils.get_global.assert_called_once_with(
|
||||
mock_envutils.ENV_VERIFICATION)
|
||||
|
||||
fake_data = []
|
||||
calls = [mock.call(uuid1),
|
||||
mock.call(uuid2)]
|
||||
mock_verification.get.assert_has_calls(calls, True)
|
||||
mock_json_dumps.assert_called_once_with(fake_data, sort_keys=True,
|
||||
indent=4)
|
||||
|
||||
@mock.patch("rally.api.Verification.get",
|
||||
side_effect=exceptions.NotFoundException())
|
||||
def test_compare_verification_not_found(self, mock_verification_get):
|
||||
uuid1 = "f7dc82da-31a6-4d40-bbf8-6d366d58960f"
|
||||
uuid2 = "2f8a05f3-d310-4f02-aabf-e1165aaa5f9c"
|
||||
|
||||
self.assertEqual(self.verify.compare(uuid1, uuid2, output_csv=False,
|
||||
output_html=False,
|
||||
output_json=True), 1)
|
||||
|
||||
mock_verification_get.assert_called_once_with(uuid1)
|
||||
|
||||
@mock.patch("rally.cli.commands.verify.open",
|
||||
side_effect=mock.mock_open(), create=True)
|
||||
@mock.patch("rally.api.Verification")
|
||||
def test_compare_with_output_csv_and_output_file(
|
||||
self, mock_verification, mock_open):
|
||||
mock_verification.get.return_value.get_results.return_value = {
|
||||
"test_cases": {}}
|
||||
|
||||
fake_string = "Type,Field,Value 1,Value 2,Test Name\r\n"
|
||||
uuid1 = "5e744557-4c3a-414f-9afb-7d3d8708028f"
|
||||
uuid2 = "efe1c74d-a632-476e-bb6a-55a9aa9cf76b"
|
||||
self.verify.compare(uuid1, uuid2, output_file="results",
|
||||
output_csv=True, output_html=False,
|
||||
output_json=False)
|
||||
|
||||
calls = [mock.call(uuid1),
|
||||
mock.call(uuid2)]
|
||||
mock_verification.get.assert_has_calls(calls, True)
|
||||
mock_open.assert_called_once_with("results", "wb")
|
||||
mock_open.side_effect().write.assert_called_once_with(fake_string)
|
||||
|
||||
@mock.patch("rally.cli.commands.verify.open",
|
||||
side_effect=mock.mock_open(), create=True)
|
||||
@mock.patch("rally.api.Verification")
|
||||
def test_compare_with_output_json_and_output_file(
|
||||
self, mock_verification, mock_open):
|
||||
mock_verification.get.return_value.get_results.return_value = {
|
||||
"test_cases": {}}
|
||||
|
||||
fake_json_string = "[]"
|
||||
uuid1 = "0505e33a-738d-4474-a611-9db21547d863"
|
||||
uuid2 = "b1908417-934e-481c-8d23-bc0badad39ed"
|
||||
self.verify.compare(uuid1, uuid2, output_file="results",
|
||||
output_csv=False, output_html=False,
|
||||
output_json=True)
|
||||
|
||||
calls = [mock.call(uuid1),
|
||||
mock.call(uuid2)]
|
||||
mock_verification.get.assert_has_calls(calls, True)
|
||||
mock_open.assert_called_once_with("results", "wb")
|
||||
mock_open.side_effect().write.assert_called_once_with(fake_json_string)
|
||||
|
||||
@mock.patch("rally.cli.commands.verify.open",
|
||||
side_effect=mock.mock_open(), create=True)
|
||||
@mock.patch("rally.api.Verification")
|
||||
@mock.patch("rally.verification.tempest.compare2html.create_report",
|
||||
return_value="")
|
||||
def test_compare_with_output_html_and_output_file(
|
||||
self, mock_compare2html_create_report,
|
||||
mock_verification, mock_open):
|
||||
mock_verification.get.return_value.get_results.return_value = {
|
||||
"test_cases": {}}
|
||||
|
||||
uuid1 = "cdf64228-77e9-414d-9d4b-f65e9d62c61f"
|
||||
uuid2 = "39393eec-1b45-4103-8ec1-631edac4b8f0"
|
||||
|
||||
fake_data = []
|
||||
self.verify.compare(uuid1, uuid2,
|
||||
output_file="results",
|
||||
output_csv=False, output_html=True,
|
||||
output_json=False)
|
||||
calls = [mock.call(uuid1),
|
||||
mock.call(uuid2)]
|
||||
mock_verification.get.assert_has_calls(calls, True)
|
||||
mock_compare2html_create_report.assert_called_once_with(fake_data)
|
||||
|
||||
mock_open.assert_called_once_with("results", "wb")
|
||||
mock_open.side_effect().write.assert_called_once_with("")
|
||||
def test_compare(self):
|
||||
self.assertEqual(1, self.verify.compare())
|
||||
|
||||
@mock.patch("rally.common.fileutils._rewrite_env_file")
|
||||
@mock.patch("rally.api.Verification.get")
|
||||
|
||||
105
tests/unit/ui/test_report.py
Normal file
105
tests/unit/ui/test_report.py
Normal file
@ -0,0 +1,105 @@
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import collections
|
||||
|
||||
import mock
|
||||
|
||||
from rally.ui import report
|
||||
from tests.unit import test
|
||||
|
||||
|
||||
class VerificationReportTestCase(test.TestCase):
|
||||
|
||||
def gen_instance(self, runs=None, uuids=None, tests=None):
|
||||
ins = report.VerificationReport({})
|
||||
ins._runs = runs or {}
|
||||
ins._uuids = uuids or list(ins._runs.keys())
|
||||
ins._tests = tests or []
|
||||
return ins
|
||||
|
||||
def test___init__(self):
|
||||
verifications = collections.OrderedDict([
|
||||
("a_uuid", {
|
||||
"tests": {"spam": {"status": "fail",
|
||||
"duration": 4.2,
|
||||
"details": "Some error",
|
||||
"tags": ["a-tag", "id-tag", "z-tag"]}}}),
|
||||
("b_uuid", {
|
||||
"tests": {"foo": {"status": "success", "duration": 0,
|
||||
"details": None,
|
||||
"tags": ["a-tag", "id-tag", "z-tag"]},
|
||||
"bar": {"status": "skip", "duration": 4.2,
|
||||
"details": None,
|
||||
"tags": ["a-tag", "id-tag", "z-tag"]}}})])
|
||||
ins = report.VerificationReport(verifications)
|
||||
self.assertEqual(verifications, ins._runs)
|
||||
self.assertEqual(["a_uuid", "b_uuid"], ins._uuids)
|
||||
tests = [
|
||||
{"has_details": False, "by_verification": {
|
||||
"b_uuid": {"duration": 4.2, "status": "skip",
|
||||
"details": None}},
|
||||
"name": "bar", "tags": ["id-tag", "a-tag", "z-tag"]},
|
||||
{"has_details": False, "by_verification": {
|
||||
"b_uuid": {"duration": 0, "status": "success",
|
||||
"details": None}},
|
||||
"name": "foo", "tags": ["id-tag", "a-tag", "z-tag"]},
|
||||
{"has_details": True, "by_verification": {
|
||||
"a_uuid": {"duration": 4.2, "status": "fail",
|
||||
"details": "Some error"}},
|
||||
"name": "spam", "tags": ["id-tag", "a-tag", "z-tag"]}]
|
||||
self.assertEqual(tests, sorted(ins._tests, key=lambda i: i["name"]))
|
||||
|
||||
@mock.patch("rally.ui.report.utils")
|
||||
@mock.patch("rally.ui.report.json.dumps", return_value="json!")
|
||||
def test_to_html(self, mock_dumps, mock_utils):
|
||||
mock_render = mock.Mock(return_value="HTML")
|
||||
mock_utils.get_template.return_value.render = mock_render
|
||||
ins = self.gen_instance(runs="runs!", uuids="uuids!", tests="tests!")
|
||||
self.assertEqual("HTML", ins.to_html())
|
||||
mock_utils.get_template.assert_called_once_with(
|
||||
"verification/report.html")
|
||||
mock_dumps.assert_called_once_with(
|
||||
{"tests": "tests!", "uuids": "uuids!", "verifications": "runs!"})
|
||||
mock_render.assert_called_once_with(data="json!", include_libs=False)
|
||||
|
||||
@mock.patch("rally.ui.report.json.dumps", return_value="json!")
|
||||
def test_to_json(self, mock_dumps):
|
||||
ins = self.gen_instance(tests="tests!")
|
||||
self.assertEqual("json!", ins.to_json())
|
||||
mock_dumps.assert_called_once_with("tests!", indent=4)
|
||||
|
||||
@mock.patch("rally.ui.report.csv")
|
||||
@mock.patch("rally.ui.report.io.BytesIO")
|
||||
def test_to_csv(self, mock_bytes_io, mock_csv):
|
||||
ins = self.gen_instance(
|
||||
uuids=["foo", "bar"],
|
||||
tests=[{"name": "test-1", "tags": ["tag1", "tag2"],
|
||||
"has_details": False,
|
||||
"by_verification": {
|
||||
"foo": {"status": "success", "duration": 1.2}}},
|
||||
{"name": "test-2", "tags": ["tag3", "tag4"],
|
||||
"has_details": False,
|
||||
"by_verification": {
|
||||
"bar": {"status": "success", "duration": 3.4}}}])
|
||||
mock_stream = mock.Mock()
|
||||
mock_stream.getvalue.return_value = "CSV!"
|
||||
mock_bytes_io.return_value.__enter__.return_value = mock_stream
|
||||
self.assertEqual("CSV!", ins.to_csv())
|
||||
mock_csv.writer.assert_called_once_with(mock_stream)
|
||||
|
||||
# Custom kwargs
|
||||
mock_csv.writer.reset_mock()
|
||||
self.assertEqual("CSV!", ins.to_csv(foo="bar"))
|
||||
mock_csv.writer.assert_called_once_with(mock_stream, foo="bar")
|
||||
@ -1,48 +0,0 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import mock
|
||||
|
||||
from rally.verification.tempest import compare2html
|
||||
from tests.unit import test
|
||||
|
||||
|
||||
class Compare2HtmlTestCase(test.TestCase):
|
||||
|
||||
@mock.patch("rally.ui.utils.get_template")
|
||||
def test_main(self, mock_get_template):
|
||||
results = [{"val2": 0.0111, "field": u"time", "val1": 0.0222,
|
||||
"type": "CHANGED", "test_name": u"test.one"},
|
||||
{"val2": 0.111, "field": u"time", "val1": 0.222,
|
||||
"type": "CHANGED", "test_name": u"test.two"},
|
||||
{"val2": 1.11, "field": u"time", "val1": 2.22,
|
||||
"type": "CHANGED", "test_name": u"test.three"}]
|
||||
|
||||
fake_template_kw = {
|
||||
"heading": {
|
||||
"title": compare2html.__title__,
|
||||
"description": compare2html.__description__,
|
||||
"parameters": [("Difference Count", len(results))]
|
||||
},
|
||||
"generator": "compare2html %s" % compare2html.__version__,
|
||||
"results": results
|
||||
}
|
||||
|
||||
template_mock = mock.MagicMock()
|
||||
mock_get_template.return_value = template_mock
|
||||
output_mock = mock.MagicMock()
|
||||
template_mock.render.return_value = output_mock
|
||||
compare2html.create_report(results)
|
||||
|
||||
mock_get_template.assert_called_once_with("verification/compare.mako")
|
||||
template_mock.render.assert_called_once_with(**fake_template_kw)
|
||||
output_mock.encode.assert_called_once_with("utf8")
|
||||
@ -1,107 +0,0 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from rally.verification.tempest import diff
|
||||
from tests.unit import test
|
||||
|
||||
|
||||
class DiffTestCase(test.TestCase):
|
||||
|
||||
def test_main(self):
|
||||
results1 = {"test.NONE": {"name": "test.NONE",
|
||||
"output": "test.NONE",
|
||||
"status": "SKIPPED",
|
||||
"time": 0.000},
|
||||
"test.zerofive": {"name": "test.zerofive",
|
||||
"output": "test.zerofive",
|
||||
"status": "FAILED",
|
||||
"time": 0.05},
|
||||
"test.one": {"name": "test.one",
|
||||
"output": "test.one",
|
||||
"status": "OK",
|
||||
"time": 0.111},
|
||||
"test.two": {"name": "test.two",
|
||||
"output": "test.two",
|
||||
"status": "OK",
|
||||
"time": 0.222},
|
||||
"test.three": {"name": "test.three",
|
||||
"output": "test.three",
|
||||
"status": "FAILED",
|
||||
"time": 0.333},
|
||||
"test.four": {"name": "test.four",
|
||||
"output": "test.four",
|
||||
"status": "OK",
|
||||
"time": 0.444},
|
||||
"test.five": {"name": "test.five",
|
||||
"output": "test.five",
|
||||
"status": "OK",
|
||||
"time": 0.555}
|
||||
}
|
||||
|
||||
results2 = {"test.one": {"name": "test.one",
|
||||
"output": "test.one",
|
||||
"status": "FAIL",
|
||||
"time": 0.1111},
|
||||
"test.two": {"name": "test.two",
|
||||
"output": "test.two",
|
||||
"status": "OK",
|
||||
"time": 0.222},
|
||||
"test.three": {"name": "test.three",
|
||||
"output": "test.three",
|
||||
"status": "OK",
|
||||
"time": 0.3333},
|
||||
"test.four": {"name": "test.four",
|
||||
"output": "test.four",
|
||||
"status": "FAIL",
|
||||
"time": 0.4444},
|
||||
"test.five": {"name": "test.five",
|
||||
"output": "test.five",
|
||||
"status": "OK",
|
||||
"time": 0.555},
|
||||
"test.six": {"name": "test.six",
|
||||
"output": "test.six",
|
||||
"status": "OK",
|
||||
"time": 0.666},
|
||||
"test.seven": {"name": "test.seven",
|
||||
"output": "test.seven",
|
||||
"status": "OK",
|
||||
"time": 0.777}
|
||||
}
|
||||
|
||||
diff_ = diff.Diff(results1, results2, 0)
|
||||
assert len(diff_.diffs) == 10
|
||||
assert len([test for test in diff_.diffs
|
||||
if test["type"] == "removed_test"]) == 2
|
||||
assert len([test for test in diff_.diffs
|
||||
if test["type"] == "new_test"]) == 2
|
||||
assert len([test for test in diff_.diffs
|
||||
if test["type"] == "value_changed"]) == 6
|
||||
assert diff_.to_csv() != ""
|
||||
assert diff_.to_html() != ""
|
||||
assert diff_.to_json() != ""
|
||||
|
||||
def test_zero_values(self):
|
||||
results1 = {"test.one": {"name": "test.one",
|
||||
"output": "test.one",
|
||||
"status": "OK",
|
||||
"time": 1}}
|
||||
|
||||
results2 = {"test.one": {"name": "test.one",
|
||||
"output": "test.one",
|
||||
"status": "FAIL",
|
||||
"time": 0}}
|
||||
|
||||
# This must NOT raise ZeroDivisionError
|
||||
diff_ = diff.Diff(results1, results2, 0)
|
||||
self.assertEqual(2, len(diff_.diffs))
|
||||
diff_ = diff.Diff(results2, results1, 0)
|
||||
self.assertEqual(2, len(diff_.diffs))
|
||||
@ -1,114 +0,0 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import datetime as dt
|
||||
|
||||
import mock
|
||||
|
||||
from rally.verification.tempest import json2html
|
||||
from tests.unit import test
|
||||
|
||||
BASE = "rally.verification.tempest"
|
||||
|
||||
|
||||
class HtmlOutputTestCase(test.TestCase):
|
||||
|
||||
@mock.patch(BASE + ".json2html.ui_utils.get_template")
|
||||
def test_generate_report(self, mock_get_template):
|
||||
results = {
|
||||
"time": 22.75,
|
||||
"tests": 4,
|
||||
"success": 1,
|
||||
"skipped": 1,
|
||||
"failures": 1,
|
||||
"expected_failures": 0,
|
||||
"unexpected_success": 0,
|
||||
"test_cases": {
|
||||
"tp": {"name": "tp",
|
||||
"status": "success",
|
||||
"time": 2},
|
||||
"ts": {"name": "ts",
|
||||
"status": "skip",
|
||||
"reason": "ts_skip",
|
||||
"time": 4},
|
||||
"tf": {"name": "tf",
|
||||
"status": "fail",
|
||||
"time": 6,
|
||||
"traceback": "fail_log"}}}
|
||||
|
||||
expected_report = {
|
||||
"failures": 1,
|
||||
"success": 1,
|
||||
"skipped": 1,
|
||||
"expected_failures": 0,
|
||||
"unexpected_success": 0,
|
||||
"total": 4,
|
||||
"time": "{0} ({1} s)".format(
|
||||
dt.timedelta(seconds=23), 22.75),
|
||||
"tests": [{"name": "tf",
|
||||
"id": 0,
|
||||
"output": "fail_log",
|
||||
"status": "fail",
|
||||
"time": 6},
|
||||
{"name": "tp",
|
||||
"id": 1,
|
||||
"output": "",
|
||||
"status": "success",
|
||||
"time": 2},
|
||||
{"name": "ts",
|
||||
"id": 2,
|
||||
"output": "Reason:\n ts_skip",
|
||||
"status": "skip",
|
||||
"time": 4}]}
|
||||
|
||||
json2html.generate_report(results)
|
||||
|
||||
mock_get_template.assert_called_once_with("verification/report.mako")
|
||||
mock_get_template.return_value.render.assert_called_once_with(
|
||||
report=expected_report)
|
||||
|
||||
@mock.patch(BASE + ".json2html.ui_utils.get_template")
|
||||
def test_convert_bug_id_in_reason_into_bug_link(self, mock_get_template):
|
||||
results = {
|
||||
"failures": 0,
|
||||
"success": 0,
|
||||
"skipped": 1,
|
||||
"expected_failures": 0,
|
||||
"unexpected_success": 0,
|
||||
"tests": 1,
|
||||
"time": 0,
|
||||
"test_cases": {"one_test": {
|
||||
"status": "skip",
|
||||
"name": "one_test",
|
||||
"reason": "Skipped until Bug: 666666 is resolved.",
|
||||
"time": "time"}}}
|
||||
|
||||
expected_report = {
|
||||
"failures": 0,
|
||||
"success": 0,
|
||||
"skipped": 1,
|
||||
"expected_failures": 0,
|
||||
"unexpected_success": 0,
|
||||
"total": 1,
|
||||
"time": "{0} ({1} s)".format(dt.timedelta(seconds=0), 0),
|
||||
"tests": [{
|
||||
"id": 0,
|
||||
"status": "skip",
|
||||
"name": "one_test",
|
||||
"output": "Reason:\n Skipped until Bug: <a href='https://"
|
||||
"launchpad.net/bugs/666666'>666666</a> is resolved.",
|
||||
"time": "time"}]}
|
||||
|
||||
json2html.generate_report(results)
|
||||
mock_get_template.assert_called_once_with("verification/report.mako")
|
||||
mock_get_template.return_value.render.assert_called_once_with(
|
||||
report=expected_report)
|
||||
Loading…
Reference in New Issue
Block a user