Merge "Add ability to compare two verification results"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
fa7d92635f
@ -32,7 +32,7 @@ LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
def main():
|
||||
# Initialize configuation and logging.
|
||||
# Initialize configuration and logging.
|
||||
CONF(sys.argv[1:], project='rally')
|
||||
log.setup('rally')
|
||||
# Prepare application and bind to the service socket.
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
|
||||
""" Rally command: verify """
|
||||
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
|
||||
@ -30,6 +31,7 @@ from rally.i18n import _
|
||||
from rally import objects
|
||||
from rally.openstack.common import cliutils as common_cliutils
|
||||
from rally.orchestrator import api
|
||||
from rally.verification.verifiers.tempest import diff
|
||||
from rally.verification.verifiers.tempest import json2html
|
||||
|
||||
|
||||
@ -71,30 +73,30 @@ class VerifyCommands(object):
|
||||
return (1)
|
||||
verification = api.verify(deploy_id, set_name, regex, tempest_config)
|
||||
if do_use:
|
||||
use.UseCommands().verification(verification['uuid'])
|
||||
use.UseCommands().verification(verification["uuid"])
|
||||
|
||||
def list(self):
|
||||
"""Display all verifications table, started and finished."""
|
||||
|
||||
fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', 'Failures',
|
||||
'Created at', 'Status']
|
||||
fields = ["UUID", "Deployment UUID", "Set name", "Tests", "Failures",
|
||||
"Created at", "Status"]
|
||||
verifications = db.verification_list()
|
||||
if verifications:
|
||||
common_cliutils.print_list(verifications, fields,
|
||||
sortby_index=fields.index('Created at'))
|
||||
sortby_index=fields.index("Created at"))
|
||||
else:
|
||||
print(_("There are no results from verifier. To run a verifier, "
|
||||
"use:\nrally verify start"))
|
||||
|
||||
@cliutils.args('--uuid', type=str, dest='verification_uuid',
|
||||
help='UUID of the verification')
|
||||
@cliutils.args('--html', action='store_true', dest='output_html',
|
||||
help=('Results will be in html format'))
|
||||
@cliutils.args('--json', action='store_true', dest='output_json',
|
||||
help=('Results will be in json format'))
|
||||
@cliutils.args('--output-file', type=str, required=False,
|
||||
dest='output_file',
|
||||
help='If specified, output will be saved to given file')
|
||||
@cliutils.args("--uuid", type=str, dest="verification_uuid",
|
||||
help="UUID of the verification")
|
||||
@cliutils.args("--html", action="store_true", dest="output_html",
|
||||
help=("Results will be in html format"))
|
||||
@cliutils.args("--json", action="store_true", dest="output_json",
|
||||
help=("Results will be in json format"))
|
||||
@cliutils.args("--output-file", type=str, required=False,
|
||||
dest="output_file",
|
||||
help="If specified, output will be saved to given file")
|
||||
@envutils.with_default_verification_id
|
||||
def results(self, verification_uuid=None, output_file=None,
|
||||
output_html=None, output_json=None):
|
||||
@ -107,13 +109,13 @@ class VerifyCommands(object):
|
||||
"""
|
||||
|
||||
try:
|
||||
results = db.verification_result_get(verification_uuid)['data']
|
||||
results = db.verification_result_get(verification_uuid)["data"]
|
||||
except exceptions.NotFoundException as e:
|
||||
print(six.text_type(e))
|
||||
return 1
|
||||
|
||||
result = ''
|
||||
if len(filter(lambda x: bool(x), [output_json, output_html])) > 1:
|
||||
result = ""
|
||||
if output_json + output_html > 1:
|
||||
print("Please specify only one output format.")
|
||||
elif output_html:
|
||||
result = json2html.main(results)
|
||||
@ -122,24 +124,24 @@ class VerifyCommands(object):
|
||||
|
||||
if output_file:
|
||||
output_file = os.path.expanduser(output_file)
|
||||
with open(output_file, 'wb') as f:
|
||||
with open(output_file, "wb") as f:
|
||||
f.write(result)
|
||||
else:
|
||||
print(result)
|
||||
|
||||
@cliutils.args('--uuid', dest='verification_uuid', type=str,
|
||||
@cliutils.args("--uuid", dest="verification_uuid", type=str,
|
||||
required=False,
|
||||
help='UUID of a verification')
|
||||
@cliutils.args('--sort-by', dest='sort_by', type=str, required=False,
|
||||
help='Tests can be sorted by "name" or "duration"')
|
||||
@cliutils.args('--detailed', dest='detailed', action='store_true',
|
||||
required=False, help='Prints traceback of failed tests')
|
||||
help="UUID of a verification")
|
||||
@cliutils.args("--sort-by", dest="sort_by", type=str, required=False,
|
||||
help="Tests can be sorted by 'name' or 'duration'")
|
||||
@cliutils.args("--detailed", dest="detailed", action="store_true",
|
||||
required=False, help="Prints traceback of failed tests")
|
||||
@envutils.with_default_verification_id
|
||||
def show(self, verification_uuid=None, sort_by='name', detailed=False):
|
||||
def show(self, verification_uuid=None, sort_by="name", detailed=False):
|
||||
"""Display results table of the verification."""
|
||||
|
||||
try:
|
||||
sortby_index = ('name', 'duration').index(sort_by)
|
||||
sortby_index = ("name", "duration").index(sort_by)
|
||||
except ValueError:
|
||||
print("Sorry, but verification results can't be sorted "
|
||||
"by '%s'." % sort_by)
|
||||
@ -153,42 +155,104 @@ class VerifyCommands(object):
|
||||
return 1
|
||||
|
||||
print ("Total results of verification:\n")
|
||||
total_fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests',
|
||||
'Failures', 'Created at', 'Status']
|
||||
total_fields = ["UUID", "Deployment UUID", "Set name", "Tests",
|
||||
"Failures", "Created at", "Status"]
|
||||
common_cliutils.print_list([verification], fields=total_fields)
|
||||
|
||||
print ("\nTests:\n")
|
||||
fields = ['name', 'time', 'status']
|
||||
fields = ["name", "time", "status"]
|
||||
|
||||
values = map(objects.Verification,
|
||||
six.itervalues(tests.data['test_cases']))
|
||||
six.itervalues(tests.data["test_cases"]))
|
||||
common_cliutils.print_list(values, fields, sortby_index=sortby_index)
|
||||
|
||||
if detailed:
|
||||
for test in six.itervalues(tests.data['test_cases']):
|
||||
if test['status'] == 'FAIL':
|
||||
for test in six.itervalues(tests.data["test_cases"]):
|
||||
if test["status"] == "FAIL":
|
||||
formatted_test = (
|
||||
'====================================================='
|
||||
'=================\n'
|
||||
'FAIL: %(name)s\n'
|
||||
'Time: %(time)s\n'
|
||||
'Type: %(type)s\n'
|
||||
'-----------------------------------------------------'
|
||||
'-----------------\n'
|
||||
'%(log)s\n'
|
||||
"====================================================="
|
||||
"=================\n"
|
||||
"FAIL: %(name)s\n"
|
||||
"Time: %(time)s\n"
|
||||
"Type: %(type)s\n"
|
||||
"-----------------------------------------------------"
|
||||
"-----------------\n"
|
||||
"%(log)s\n"
|
||||
) % {
|
||||
'name': test['name'], 'time': test['time'],
|
||||
'type': test['failure']['type'],
|
||||
'log': test['failure']['log']}
|
||||
"name": test["name"], "time": test["time"],
|
||||
"type": test["failure"]["type"],
|
||||
"log": test["failure"]["log"]}
|
||||
print (formatted_test)
|
||||
|
||||
@cliutils.args('--uuid', dest='verification_uuid', type=str,
|
||||
@cliutils.args("--uuid", dest="verification_uuid", type=str,
|
||||
required=False,
|
||||
help='UUID of a verification')
|
||||
@cliutils.args('--sort-by', dest='sort_by', type=str, required=False,
|
||||
help='Tests can be sorted by "name" or "duration"')
|
||||
help="UUID of a verification")
|
||||
@cliutils.args("--sort-by", dest="sort_by", type=str, required=False,
|
||||
help="Tests can be sorted by 'name' or 'duration'")
|
||||
@envutils.with_default_verification_id
|
||||
def detailed(self, verification_uuid=None, sort_by='name'):
|
||||
def detailed(self, verification_uuid=None, sort_by="name"):
|
||||
"""Display results table of verification with detailed errors."""
|
||||
|
||||
self.show(verification_uuid, sort_by, True)
|
||||
|
||||
@cliutils.args("--uuid-1", type=str, dest="uuid1",
|
||||
help="UUID of the first verification")
|
||||
@cliutils.args("--uuid-2", type=str, dest="uuid2",
|
||||
help="UUID of the second verification")
|
||||
@cliutils.args("--csv", action="store_true", dest="output_csv",
|
||||
help=("Save results in csv format to specified file"))
|
||||
@cliutils.args("--html", action="store_true", dest="output_html",
|
||||
help=("Save results in html format to specified file"))
|
||||
@cliutils.args("--json", action="store_true", dest="output_json",
|
||||
help=("Save results in json format to specified file"))
|
||||
@cliutils.args("--output-file", type=str, required=False,
|
||||
dest="output_file",
|
||||
help="If specified, output will be saved to given file")
|
||||
@cliutils.args("--threshold", type=int, required=False,
|
||||
dest="threshold", default=0,
|
||||
help="If specified, timing differences must exceed this "
|
||||
"percentage threshold to be included in output")
|
||||
def compare(self, uuid1=None, uuid2=None,
|
||||
output_file=None, output_csv=None, output_html=None,
|
||||
output_json=None, threshold=0):
|
||||
"""Compare two verification results.
|
||||
|
||||
:param uuid1: First Verification UUID
|
||||
:param uuid2: Second Verification UUID
|
||||
:param output_file: If specified, output will be saved to given file
|
||||
:param output_csv: Save results in csv format to the specified file
|
||||
:param output_html: Save results in html format to the specified file
|
||||
:param output_json: Save results in json format to the specified file
|
||||
(Default)
|
||||
:param threshold: Timing difference threshold percentage
|
||||
"""
|
||||
|
||||
try:
|
||||
results1 = db.verification_result_get(uuid1)["data"]["test_cases"]
|
||||
results2 = db.verification_result_get(uuid2)["data"]["test_cases"]
|
||||
_diff = diff.Diff(results1, results2, threshold)
|
||||
except exceptions.NotFoundException as e:
|
||||
print(six.text_type(e))
|
||||
return 1
|
||||
|
||||
result = ""
|
||||
if output_json + output_html + output_csv > 1:
|
||||
print("Please specify only one output format, either --json, "
|
||||
"--html or --csv.")
|
||||
return 1
|
||||
elif output_html:
|
||||
result = _diff.to_html()
|
||||
elif output_csv:
|
||||
result = _diff.to_csv()
|
||||
else:
|
||||
result = _diff.to_json()
|
||||
|
||||
if output_file:
|
||||
with open(output_file, "wb") as f:
|
||||
if output_csv:
|
||||
writer = csv.writer(f, dialect="excel")
|
||||
writer.writerows(result)
|
||||
else:
|
||||
f.write(result)
|
||||
else:
|
||||
print(result)
|
||||
|
||||
41
rally/verification/verifiers/tempest/compare2html.py
Normal file
41
rally/verification/verifiers/tempest/compare2html.py
Normal file
@ -0,0 +1,41 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""Output verification comparison results in html."""
|
||||
|
||||
import os
|
||||
|
||||
import mako.template
|
||||
|
||||
__description__ = "List differences between two verification runs"
|
||||
__title__ = "Verification Comparison"
|
||||
__version__ = "0.1"
|
||||
|
||||
|
||||
def create_report(results):
|
||||
template_kw = {
|
||||
"heading": {
|
||||
"title": __title__,
|
||||
"description": __description__,
|
||||
"parameters": [("Difference Count", len(results))]
|
||||
},
|
||||
"generator": "compare2html %s" % __version__,
|
||||
"results": results
|
||||
}
|
||||
|
||||
template_path = os.path.join(os.path.dirname(__file__),
|
||||
"report_templates",
|
||||
"compare.mako")
|
||||
|
||||
with open(template_path) as f:
|
||||
template = mako.template.Template(f.read(), strict_undefined=True)
|
||||
output = template.render(**template_kw)
|
||||
return output.encode('utf8')
|
||||
108
rally/verification/verifiers/tempest/diff.py
Normal file
108
rally/verification/verifiers/tempest/diff.py
Normal file
@ -0,0 +1,108 @@
|
||||
# Copyright 2014 Dell Inc.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import json
|
||||
|
||||
import compare2html
|
||||
|
||||
|
||||
class Diff(object):
|
||||
|
||||
def __init__(self, test_cases1, test_cases2, threshold):
|
||||
"""Compare two verification results.
|
||||
|
||||
Compares two verification results and emits
|
||||
desired output, csv, html, json or pprint.
|
||||
|
||||
:param test_cases1: older verification json
|
||||
:param test_cases2: newer verification json
|
||||
:param threshold: test time difference percentage threshold
|
||||
|
||||
"""
|
||||
self.threshold = threshold
|
||||
self.diffs = self._compare(test_cases1, test_cases2)
|
||||
|
||||
def _compare(self, tc1, tc2):
|
||||
"""Compare two verification results.
|
||||
|
||||
:param tc1: first verification test cases json
|
||||
:param tc2: second verification test cases json
|
||||
|
||||
Typical test case json schema:
|
||||
"test_case_key": {
|
||||
"failure": {
|
||||
"log": ""
|
||||
},
|
||||
"name": "",
|
||||
"output": "",
|
||||
"status": "",
|
||||
"time": 0.0
|
||||
}
|
||||
"""
|
||||
names1 = sorted(tc1)
|
||||
names2 = sorted(tc2)
|
||||
|
||||
diffs = []
|
||||
i = j = 0
|
||||
while i < len(names1) and j < len(names2):
|
||||
name1 = names1[i] if i < len(names1) else None
|
||||
name2 = names2[j] if j < len(names2) else None
|
||||
if name1 and name2 and name1 == name2:
|
||||
diffs.extend(self._diff_values(name1, tc1[name1], tc2[name2]))
|
||||
i += 1
|
||||
j += 1
|
||||
|
||||
elif (not name1) or (name1 > name2):
|
||||
diffs.append({"type": "new_test", "test_name": name2})
|
||||
j += 1
|
||||
else:
|
||||
diffs.append({"type": "removed_test", "test_name": name1})
|
||||
i += 1
|
||||
|
||||
return diffs
|
||||
|
||||
def _diff_values(self, name, result1, result2):
|
||||
th = self.threshold
|
||||
fields = ["status", "time", "output"]
|
||||
diffs = []
|
||||
for field in fields:
|
||||
val1 = result1[field]
|
||||
val2 = result2[field]
|
||||
if val1 != val2 and not (field == "time"
|
||||
and abs(((val2 - val1) / val1) * 100)
|
||||
< th):
|
||||
diffs.append({
|
||||
"field": field,
|
||||
"type": "value_changed",
|
||||
"test_name": name,
|
||||
"val1": val1,
|
||||
"val2": val2
|
||||
})
|
||||
return diffs
|
||||
|
||||
def to_csv(self):
|
||||
rows = (("Type", "Field", "Value 1", "Value 2", "Test Name"),)
|
||||
for res in self.diffs:
|
||||
row = (res.get("type"), res.get("field", ""),
|
||||
res.get("val1", ""), res.get("val2", ""),
|
||||
res.get("test_name"))
|
||||
rows = rows + (row,)
|
||||
return rows
|
||||
|
||||
def to_json(self):
|
||||
return json.dumps(self.diffs, sort_keys=True, indent=4)
|
||||
|
||||
def to_html(self):
|
||||
return compare2html.create_report(self.diffs)
|
||||
@ -0,0 +1,165 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>${heading["title"]}</title>
|
||||
<meta name="generator" content="${generator}">
|
||||
<meta charset="utf-8">
|
||||
<script type="text/javascript">
|
||||
var DOWN = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAGCAYAAAAVMmT4AAAAJUlEQVQYlWNgYGD4TwJmYCBFIYYGFhYWvArx2YAXEK0QWQMGAADd8SPpeGzm9QAAAABJRU5ErkJggg==";
|
||||
var NONE = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAGCAYAAAAVMmT4AAAADUlEQVQYlWNgGAUIAAABDgAB6WzgmwAAAABJRU5ErkJggg==";
|
||||
var UP = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAGCAYAAAAVMmT4AAAAK0lEQVQYlWNgwA7+4xDHqhCGiVaIVwNcAQsLC14N2EzEqoEYhf8ZGBj+AwCZbyPp8zIdEAAAAABJRU5ErkJggg==";
|
||||
|
||||
function sort_table(table_id, col, sort){
|
||||
var table = document.getElementById(table_id);
|
||||
var tbody = table.tBodies[0];
|
||||
var header_row = table.tHead.rows[0];
|
||||
render_header(col, sort, header_row);
|
||||
sort_results(tbody, col, sort);
|
||||
}
|
||||
|
||||
function render_header(col, sort, header_row){
|
||||
var h_cells = header_row.cells;
|
||||
for(i = 0; i < h_cells.length; i++){
|
||||
var cell = h_cells[i];
|
||||
var img = cell.firstElementChild;
|
||||
if (i == col){
|
||||
if (sort == 1){
|
||||
img.src = UP;
|
||||
}else{
|
||||
img.src = DOWN;
|
||||
}
|
||||
}else{ //spacer image
|
||||
img.src = NONE;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sort_results(tbody, col, sort) {
|
||||
var rows = tbody.rows, rlen = rows.length, arr = new Array(), i, j, cells, clen;
|
||||
// fill the array with values from the table
|
||||
for(i = 0; i < rlen; i++){
|
||||
cells = rows[i].cells;
|
||||
clen = cells.length;
|
||||
arr[i] = new Array();
|
||||
for(j = 0; j < clen; j++){
|
||||
arr[i][j] = cells[j].innerHTML;
|
||||
}
|
||||
}
|
||||
// sort the array by the specified column number (col) and order (sort)
|
||||
arr.sort(function(a, b){
|
||||
return (a[col] == b[col]) ? 0 : ((a[col] > b[col]) ? sort : -1*sort);
|
||||
});
|
||||
for(i = 0; i < rlen; i++){
|
||||
arr[i] = "<td>"+arr[i].join("</td><td>")+"</td>";
|
||||
}
|
||||
tbody.innerHTML = "<tr>"+arr.join("</tr><tr>")+"</tr>";
|
||||
}
|
||||
</script>
|
||||
<style type="text/css" media="screen">
|
||||
body {
|
||||
font-family: verdana, arial, helvetica, sans-serif;
|
||||
font-size: 80%;
|
||||
}
|
||||
|
||||
table {
|
||||
font-size: 100%; width: 100%;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 16pt;
|
||||
color: gray;
|
||||
}
|
||||
|
||||
.heading {
|
||||
margin-top: 0ex;
|
||||
margin-bottom: 1ex;
|
||||
}
|
||||
|
||||
.heading .attribute {
|
||||
margin-top: 1ex;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.heading .description {
|
||||
margin-top: 4ex;
|
||||
margin-bottom: 6ex;
|
||||
}
|
||||
|
||||
#results_table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
border: 1px solid #777;
|
||||
}
|
||||
|
||||
#header_row {
|
||||
font-weight: bold;
|
||||
color: white;
|
||||
background-color: #777;
|
||||
}
|
||||
|
||||
#results_table td {
|
||||
border: 1px solid #777;
|
||||
padding: 2px;
|
||||
}
|
||||
|
||||
.testcase { margin-left: 2em;}
|
||||
|
||||
img.updown{
|
||||
padding-left: 3px;
|
||||
padding-bottom: 2px;
|
||||
}
|
||||
|
||||
th:hover{
|
||||
cursor:pointer;
|
||||
}
|
||||
|
||||
.nowrap {white-space: nowrap;}
|
||||
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div class="heading">
|
||||
<h1>${heading["title"]}</h1>
|
||||
% for name, value in heading["parameters"]:
|
||||
<p class="attribute"><strong>${name}:</strong> ${value}</p>
|
||||
% endfor
|
||||
<p class="description">${heading["description"]}</p>
|
||||
</div>
|
||||
|
||||
<table id="results_table">
|
||||
<colgroup>
|
||||
<col align="left" />
|
||||
<col align="left" />
|
||||
<col align="left" />
|
||||
<col align="left" />
|
||||
<col align="left" />
|
||||
</colgroup>
|
||||
<thead>
|
||||
<tr id="header_row">
|
||||
<th class="nowrap" onclick="sort_table('results_table', 0, col1_sort); col1_sort *= -1; col2_sort = 1; col3_sort = 1; col4_sort = 1; col5_sort = 1;">Type<img class="updown" src=NONE /></th>
|
||||
<th class="nowrap" onclick="sort_table('results_table', 1, col2_sort); col2_sort *= -1; col1_sort = 1; col3_sort = 1; col4_sort = 1; col5_sort = 1;">Field<img class="updown" src=NONE /></th>
|
||||
<th class="nowrap" onclick="sort_table('results_table', 2, col3_sort); col3_sort *= -1; col1_sort = 1; col2_sort = 1; col4_sort = 1; col5_sort = 1;">Value 1<img class="updown" src=NONE /></th>
|
||||
<th class="nowrap" onclick="sort_table('results_table', 3, col4_sort); col4_sort *= -1; col1_sort = 1; col2_sort = 1; col3_sort = 1; col5_sort = 1;">Value 2<img class="updown" src=NONE /></th>
|
||||
<th onclick="sort_table('results_table', 4, col5_sort); col5_sort *= -1; col1_sort = 1; col2_sort = 1; col3_sort = 1; col4_sort = 1;">Test Name<img class="updown" src=NONE /></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="results">
|
||||
% for diff in results:
|
||||
<tr class="">
|
||||
<td class="type">${diff.get("type")}</td>
|
||||
<td class="field">${diff.get("field", "")}</td>
|
||||
<td class="val">${diff.get("val1", "")}</td>
|
||||
<td class="val">${diff.get("val2", "")}</td>
|
||||
<td class="testname">${diff.get("test_name")}</td>
|
||||
</tr>
|
||||
% endfor
|
||||
</table>
|
||||
<script type="text/javascript">
|
||||
var col1_sort = 1, col2_sort = 1, col3_sort = 1; col4_sort = 1; col5_sort = 1;
|
||||
sort_table("results_table", 4, col5_sort);
|
||||
col5_sort *= -1;
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@ -31,41 +31,41 @@ class VerifyCommandsTestCase(test.TestCase):
|
||||
self.verify = verify.VerifyCommands()
|
||||
|
||||
self.image1 = mock.Mock()
|
||||
self.image1.name = 'cirros-1'
|
||||
self.image1.id = 'fake_image_id_1'
|
||||
self.image1.name = "cirros-1"
|
||||
self.image1.id = "fake_image_id_1"
|
||||
self.image2 = mock.Mock()
|
||||
self.image2.id = 'fake_image_id_2'
|
||||
self.image2.name = 'cirros-2'
|
||||
self.image2.id = "fake_image_id_2"
|
||||
self.image2.name = "cirros-2"
|
||||
|
||||
self.flavor1 = mock.Mock()
|
||||
self.flavor2 = mock.Mock()
|
||||
self.flavor1.id = 'fake_flavor_id_1'
|
||||
self.flavor2.id = 'fake_flavor_id_2'
|
||||
self.flavor1.id = "fake_flavor_id_1"
|
||||
self.flavor2.id = "fake_flavor_id_2"
|
||||
self.flavor1.ram = 128
|
||||
self.flavor2.ram = 64
|
||||
|
||||
@mock.patch('rally.osclients.Clients')
|
||||
@mock.patch('rally.orchestrator.api.verify')
|
||||
@mock.patch("rally.osclients.Clients")
|
||||
@mock.patch("rally.orchestrator.api.verify")
|
||||
def test_start(self, mock_verify, mock_clients):
|
||||
deploy_id = '0fba91c6-82d5-4ce1-bd00-5d7c989552d9'
|
||||
deploy_id = "0fba91c6-82d5-4ce1-bd00-5d7c989552d9"
|
||||
mock_clients().glance().images.list.return_value = [
|
||||
self.image1, self.image2]
|
||||
mock_clients().nova().flavors.list.return_value = [
|
||||
self.flavor1, self.flavor2]
|
||||
|
||||
self.verify.start(deploy_id=deploy_id)
|
||||
default_set_name = 'smoke'
|
||||
default_set_name = "smoke"
|
||||
default_regex = None
|
||||
|
||||
mock_verify.assert_called_once_with(deploy_id,
|
||||
default_set_name, default_regex,
|
||||
None)
|
||||
|
||||
@mock.patch('rally.osclients.Clients')
|
||||
@mock.patch('rally.orchestrator.api.verify')
|
||||
@mock.patch("rally.osclients.Clients")
|
||||
@mock.patch("rally.orchestrator.api.verify")
|
||||
def test_start_with_user_specified_tempest_config(self, mock_verify,
|
||||
mock_clients):
|
||||
deploy_id = '0fba91c6-82d5-4ce1-bd00-5d7c989552d9'
|
||||
deploy_id = "0fba91c6-82d5-4ce1-bd00-5d7c989552d9"
|
||||
mock_clients().glance().images.list.return_value = [
|
||||
self.image1, self.image2]
|
||||
mock_clients().nova().flavors.list.return_value = [
|
||||
@ -73,7 +73,7 @@ class VerifyCommandsTestCase(test.TestCase):
|
||||
tempest_config = tempfile.NamedTemporaryFile()
|
||||
self.verify.start(deploy_id=deploy_id,
|
||||
tempest_config=tempest_config.name)
|
||||
default_set_name = 'smoke'
|
||||
default_set_name = "smoke"
|
||||
default_regex = None
|
||||
|
||||
mock_verify.assert_called_once_with(deploy_id,
|
||||
@ -81,114 +81,212 @@ class VerifyCommandsTestCase(test.TestCase):
|
||||
tempest_config.name)
|
||||
tempest_config.close()
|
||||
|
||||
@mock.patch('rally.orchestrator.api.verify')
|
||||
@mock.patch("rally.orchestrator.api.verify")
|
||||
def test_start_with_wrong_set_name(self, mock_verify):
|
||||
deploy_id = 'f2009aae-6ef3-468e-96b2-3c987d584010'
|
||||
deploy_id = "f2009aae-6ef3-468e-96b2-3c987d584010"
|
||||
|
||||
wrong_set_name = 'unexpected_value'
|
||||
wrong_set_name = "unexpected_value"
|
||||
|
||||
self.verify.start(deploy_id, wrong_set_name)
|
||||
|
||||
self.assertNotIn(wrong_set_name, consts.TEMPEST_TEST_SETS)
|
||||
self.assertFalse(mock_verify.called)
|
||||
|
||||
@mock.patch('rally.openstack.common.cliutils.print_list')
|
||||
@mock.patch('rally.db.verification_list')
|
||||
@mock.patch("rally.openstack.common.cliutils.print_list")
|
||||
@mock.patch("rally.db.verification_list")
|
||||
def test_list(self, mock_db_verification_list, mock_print_list):
|
||||
fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', 'Failures',
|
||||
'Created at', 'Status']
|
||||
verifications = {'dummy': []}
|
||||
fields = ["UUID", "Deployment UUID", "Set name", "Tests", "Failures",
|
||||
"Created at", "Status"]
|
||||
verifications = {"dummy": []}
|
||||
mock_db_verification_list.return_value = verifications
|
||||
self.verify.list()
|
||||
mock_db_verification_list.assert_called_once_with()
|
||||
mock_print_list.assert_called_once_with(verifications, fields,
|
||||
sortby_index=fields.index(
|
||||
'Created at'))
|
||||
"Created at"))
|
||||
|
||||
@mock.patch('rally.openstack.common.cliutils.print_list')
|
||||
@mock.patch('rally.db.verification_get')
|
||||
@mock.patch('rally.db.verification_result_get')
|
||||
@mock.patch('rally.objects.Verification')
|
||||
@mock.patch("rally.openstack.common.cliutils.print_list")
|
||||
@mock.patch("rally.db.verification_get")
|
||||
@mock.patch("rally.db.verification_result_get")
|
||||
@mock.patch("rally.objects.Verification")
|
||||
def test_show(self, mock_obj_verification,
|
||||
mock_verification_result_get, mock_verification_get,
|
||||
mock_print_list):
|
||||
|
||||
class Test_dummy():
|
||||
data = {'test_cases': {'test_a': {'name': 'test_a', 'time': 20,
|
||||
'status': 'PASS'},
|
||||
'test_b': {'name': 'test_b', 'time': 20,
|
||||
'status': 'SKIP'},
|
||||
'test_c': {'name': 'test_c', 'time': 20,
|
||||
'status': 'FAIL'}}}
|
||||
data = {"test_cases": {"test_a": {"name": "test_a", "time": 20,
|
||||
"status": "PASS"},
|
||||
"test_b": {"name": "test_b", "time": 20,
|
||||
"status": "SKIP"},
|
||||
"test_c": {"name": "test_c", "time": 20,
|
||||
"status": "FAIL"}}}
|
||||
|
||||
verification_id = '39121186-b9a4-421d-b094-6c6b270cf9e9'
|
||||
total_fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests',
|
||||
'Failures', 'Created at', 'Status']
|
||||
fields = ['name', 'time', 'status']
|
||||
verification_id = "39121186-b9a4-421d-b094-6c6b270cf9e9"
|
||||
total_fields = ["UUID", "Deployment UUID", "Set name", "Tests",
|
||||
"Failures", "Created at", "Status"]
|
||||
fields = ["name", "time", "status"]
|
||||
verification = mock.MagicMock()
|
||||
tests = Test_dummy()
|
||||
mock_verification_result_get.return_value = tests
|
||||
mock_verification_get.return_value = verification
|
||||
mock_obj_verification.return_value = 1
|
||||
values = map(objects.Verification,
|
||||
six.itervalues(tests.data['test_cases']))
|
||||
six.itervalues(tests.data["test_cases"]))
|
||||
self.verify.show(verification_id)
|
||||
mock_print_list.assert_any_call(
|
||||
[verification], fields=total_fields)
|
||||
mock_print_list.assert_any_call([verification], fields=total_fields)
|
||||
mock_verification_get.assert_called_once_with(verification_id)
|
||||
mock_verification_result_get.assert_called_once_with(verification_id)
|
||||
mock_print_list.assert_any_call(values, fields, sortby_index=0)
|
||||
|
||||
@mock.patch('rally.db.verification_result_get', return_value={'data': {}})
|
||||
@mock.patch('json.dumps')
|
||||
@mock.patch("rally.db.verification_result_get", return_value={"data": {}})
|
||||
@mock.patch("json.dumps")
|
||||
def test_results(self, mock_json_dumps, mock_db_result_get):
|
||||
verification_uuid = 'a0231bdf-6a4e-4daf-8ab1-ae076f75f070'
|
||||
self.verify.results(verification_uuid, output_json=True)
|
||||
verification_uuid = "a0231bdf-6a4e-4daf-8ab1-ae076f75f070"
|
||||
self.verify.results(verification_uuid, output_html=False,
|
||||
output_json=True)
|
||||
|
||||
mock_db_result_get.assert_called_once_with(verification_uuid)
|
||||
mock_json_dumps.assert_called_once_with({}, sort_keys=True, indent=4)
|
||||
|
||||
@mock.patch('rally.db.verification_result_get')
|
||||
@mock.patch("rally.db.verification_result_get")
|
||||
def test_results_verification_not_found(self, mock_db_result_get):
|
||||
verification_uuid = '9044ced5-9c84-4666-8a8f-4b73a2b62acb'
|
||||
verification_uuid = "9044ced5-9c84-4666-8a8f-4b73a2b62acb"
|
||||
mock_db_result_get.side_effect = exceptions.NotFoundException()
|
||||
self.assertEqual(self.verify.results(verification_uuid), 1)
|
||||
self.assertEqual(self.verify.results(verification_uuid,
|
||||
output_html=False,
|
||||
output_json=True), 1)
|
||||
|
||||
mock_db_result_get.assert_called_once_with(verification_uuid)
|
||||
|
||||
@mock.patch('rally.cmd.commands.verify.open', create=True)
|
||||
@mock.patch('rally.db.verification_result_get', return_value={'data': {}})
|
||||
@mock.patch("rally.cmd.commands.verify.open", create=True)
|
||||
@mock.patch("rally.db.verification_result_get", return_value={"data": {}})
|
||||
def test_results_with_output_json_and_output_file(self,
|
||||
mock_db_result_get,
|
||||
mock_open):
|
||||
mock_open.return_value = mock.MagicMock()
|
||||
verification_uuid = '94615cd4-ff45-4123-86bd-4b0741541d09'
|
||||
self.verify.results(verification_uuid, output_file='results',
|
||||
output_json=True)
|
||||
verification_uuid = "94615cd4-ff45-4123-86bd-4b0741541d09"
|
||||
self.verify.results(verification_uuid, output_file="results",
|
||||
output_html=False, output_json=True)
|
||||
|
||||
mock_db_result_get.assert_called_once_with(verification_uuid)
|
||||
mock_open.assert_called_once_with('results', 'wb')
|
||||
mock_open.assert_called_once_with("results", "wb")
|
||||
fake_file = mock_open.return_value.__enter__.return_value
|
||||
fake_file.write.assert_called_once_with('{}')
|
||||
fake_file.write.assert_called_once_with("{}")
|
||||
|
||||
@mock.patch('rally.cmd.commands.verify.open', create=True)
|
||||
@mock.patch('rally.db.verification_result_get')
|
||||
@mock.patch('rally.verification.verifiers.tempest.json2html.main',
|
||||
return_value='')
|
||||
@mock.patch("rally.cmd.commands.verify.open", create=True)
|
||||
@mock.patch("rally.db.verification_result_get")
|
||||
@mock.patch("rally.verification.verifiers.tempest.json2html.main",
|
||||
return_value="")
|
||||
def test_results_with_output_html_and_output_file(self,
|
||||
mock_json2html_main,
|
||||
mock_db_result_get,
|
||||
mock_open):
|
||||
mock_open.return_value = mock.MagicMock()
|
||||
verification_uuid = '7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa'
|
||||
verification_uuid = "7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa"
|
||||
fake_data = {}
|
||||
results = {'data': fake_data}
|
||||
results = {"data": fake_data}
|
||||
mock_db_result_get.return_value = results
|
||||
self.verify.results(verification_uuid, output_html=True,
|
||||
output_file='results')
|
||||
output_json=False, output_file="results")
|
||||
|
||||
mock_db_result_get.assert_called_once_with(verification_uuid)
|
||||
mock_json2html_main.assert_called_once_with(fake_data)
|
||||
mock_open.assert_called_once_with('results', 'wb')
|
||||
mock_open.assert_called_once_with("results", "wb")
|
||||
fake_file = mock_open.return_value.__enter__.return_value
|
||||
fake_file.write.assert_called_once_with('')
|
||||
fake_file.write.assert_called_once_with("")
|
||||
|
||||
@mock.patch("rally.db.verification_result_get",
|
||||
return_value={"data": {"test_cases": {}}})
|
||||
@mock.patch("json.dumps")
|
||||
def test_compare(self, mock_json_dumps, mock_db_result_get):
|
||||
uuid1 = "8eda1b10-c8a4-4316-9603-8468ff1d1560"
|
||||
uuid2 = "f6ef0a98-1b18-452f-a6a7-922555c2e326"
|
||||
self.verify.compare(uuid1, uuid2, output_csv=False, output_html=False,
|
||||
output_json=True)
|
||||
|
||||
fake_data = []
|
||||
calls = [mock.call(uuid1),
|
||||
mock.call(uuid2)]
|
||||
mock_db_result_get.assert_has_calls(calls, True)
|
||||
mock_json_dumps.assert_called_once_with(fake_data, sort_keys=True,
|
||||
indent=4)
|
||||
|
||||
@mock.patch("rally.db.verification_result_get",
|
||||
side_effect=exceptions.NotFoundException())
|
||||
def test_compare_verification_not_found(self, mock_db_result_get):
|
||||
uuid1 = "f7dc82da-31a6-4d40-bbf8-6d366d58960f"
|
||||
uuid2 = "2f8a05f3-d310-4f02-aabf-e1165aaa5f9c"
|
||||
|
||||
self.assertEqual(self.verify.compare(uuid1, uuid2, output_csv=False,
|
||||
output_html=False,
|
||||
output_json=True), 1)
|
||||
|
||||
mock_db_result_get.assert_called_once_with(uuid1)
|
||||
|
||||
@mock.patch("rally.cmd.commands.verify.open", create=True)
|
||||
@mock.patch("rally.db.verification_result_get",
|
||||
return_value={"data": {"test_cases": {}}})
|
||||
def test_compare_with_output_csv_and_output_file(self,
|
||||
mock_db_result_get,
|
||||
mock_open):
|
||||
|
||||
fake_string = "Type,Field,Value 1,Value 2,Test Name\r\n"
|
||||
uuid1 = "5e744557-4c3a-414f-9afb-7d3d8708028f"
|
||||
uuid2 = "efe1c74d-a632-476e-bb6a-55a9aa9cf76b"
|
||||
self.verify.compare(uuid1, uuid2, output_file="results",
|
||||
output_csv=True, output_html=False,
|
||||
output_json=False)
|
||||
|
||||
calls = [mock.call(uuid1),
|
||||
mock.call(uuid2)]
|
||||
mock_db_result_get.assert_has_calls(calls, True)
|
||||
mock_open.assert_called_once_with("results", "wb")
|
||||
fake_file = mock_open.return_value.__enter__.return_value
|
||||
fake_file.write.assert_called_once_with(fake_string)
|
||||
|
||||
@mock.patch("rally.cmd.commands.verify.open", create=True)
|
||||
@mock.patch("rally.db.verification_result_get",
|
||||
return_value={"data": {"test_cases": {}}})
|
||||
def test_compare_with_output_json_and_output_file(self,
|
||||
mock_db_result_get,
|
||||
mock_open):
|
||||
fake_json_string = "[]"
|
||||
uuid1 = "0505e33a-738d-4474-a611-9db21547d863"
|
||||
uuid2 = "b1908417-934e-481c-8d23-bc0badad39ed"
|
||||
self.verify.compare(uuid1, uuid2, output_file="results",
|
||||
output_csv=False, output_html=False,
|
||||
output_json=True)
|
||||
|
||||
calls = [mock.call(uuid1),
|
||||
mock.call(uuid2)]
|
||||
mock_db_result_get.assert_has_calls(calls, True)
|
||||
mock_open.assert_called_once_with("results", "wb")
|
||||
fake_file = mock_open.return_value.__enter__.return_value
|
||||
fake_file.write.assert_called_once_with(fake_json_string)
|
||||
|
||||
@mock.patch("rally.cmd.commands.verify.open", create=True)
|
||||
@mock.patch("rally.db.verification_result_get")
|
||||
@mock.patch(("rally.verification.verifiers.tempest."
|
||||
"compare2html.create_report"), return_value="")
|
||||
def test_compare_with_output_html_and_output_file(self,
|
||||
mock_compare2html_create,
|
||||
mock_db_result_get,
|
||||
mock_open):
|
||||
|
||||
uuid1 = "cdf64228-77e9-414d-9d4b-f65e9d62c61f"
|
||||
uuid2 = "39393eec-1b45-4103-8ec1-631edac4b8f0"
|
||||
results = {"data": {"test_cases": {}}}
|
||||
fake_data = []
|
||||
self.verify.compare(uuid1, uuid2,
|
||||
output_file="results",
|
||||
output_csv=False, output_html=True,
|
||||
output_json=False)
|
||||
mock_db_result_get.return_value = results
|
||||
calls = [mock.call(uuid1),
|
||||
mock.call(uuid2)]
|
||||
mock_db_result_get.assert_has_calls(calls, True)
|
||||
mock_compare2html_create.assert_called_once_with(fake_data)
|
||||
|
||||
mock_open.assert_called_once_with("results", "wb")
|
||||
fake_file = mock_open.return_value.__enter__.return_value
|
||||
fake_file.write.assert_called_once_with("")
|
||||
|
||||
39
tests/unit/verification/verifiers/test_compare2html.py
Normal file
39
tests/unit/verification/verifiers/test_compare2html.py
Normal file
@ -0,0 +1,39 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import mock
|
||||
|
||||
from rally.verification.verifiers.tempest import compare2html
|
||||
from tests.unit import test
|
||||
|
||||
|
||||
class Compare2HtmlTestCase(test.TestCase):
|
||||
|
||||
def test_main(self):
|
||||
results = [{'val2': 0.0111, 'field': u'time', 'val1': 0.0222,
|
||||
'type': 'CHANGED', 'test_name': u'test.one'},
|
||||
{'val2': 0.111, 'field': u'time', 'val1': 0.222,
|
||||
'type': 'CHANGED', 'test_name': u'test.two'},
|
||||
{'val2': 1.11, 'field': u'time', 'val1': 2.22,
|
||||
'type': 'CHANGED', 'test_name': u'test.three'}]
|
||||
|
||||
fake_kw = {"heading":
|
||||
{"title": compare2html.__title__,
|
||||
"description": compare2html.__description__,
|
||||
"parameters": [("Difference Count", len(results))]
|
||||
},
|
||||
"generator": "compare2html %s" % compare2html.__version__,
|
||||
"results": results}
|
||||
|
||||
with mock.patch('mako.template.Template') as mock_mako:
|
||||
compare2html.create_report(results)
|
||||
mock_mako().render.assert_called_once_with(**fake_kw)
|
||||
76
tests/unit/verification/verifiers/test_diff.py
Normal file
76
tests/unit/verification/verifiers/test_diff.py
Normal file
@ -0,0 +1,76 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from rally.verification.verifiers.tempest import diff
|
||||
from tests.unit import test
|
||||
|
||||
|
||||
class DiffTestCase(test.TestCase):
|
||||
|
||||
def test_main(self):
|
||||
results1 = {'test.NONE': {'name': 'test.NONE',
|
||||
'output': 'test.NONE',
|
||||
'status': 'SKIPPED',
|
||||
'time': 0.000},
|
||||
'test.one': {'name': 'test.one',
|
||||
'output': 'test.one',
|
||||
'status': 'OK',
|
||||
'time': 0.111},
|
||||
'test.two': {'name': 'test.two',
|
||||
'output': 'test.two',
|
||||
'status': 'OK',
|
||||
'time': 0.222},
|
||||
'test.three': {'name': 'test.three',
|
||||
'output': 'test.three',
|
||||
'status': 'FAILED',
|
||||
'time': 0.333},
|
||||
'test.four': {'name': 'test.four',
|
||||
'output': 'test.four',
|
||||
'status': 'OK',
|
||||
'time': 0.444},
|
||||
'test.five': {'name': 'test.five',
|
||||
'output': 'test.five',
|
||||
'status': 'OK',
|
||||
'time': 0.555}
|
||||
}
|
||||
|
||||
results2 = {'test.one': {'name': 'test.one',
|
||||
'output': 'test.one',
|
||||
'status': 'FAIL',
|
||||
'time': 0.1111},
|
||||
'test.two': {'name': 'test.two',
|
||||
'output': 'test.two',
|
||||
'status': 'OK',
|
||||
'time': 0.222},
|
||||
'test.three': {'name': 'test.three',
|
||||
'output': 'test.three',
|
||||
'status': 'OK',
|
||||
'time': 0.3333},
|
||||
'test.four': {'name': 'test.four',
|
||||
'output': 'test.four',
|
||||
'status': 'FAIL',
|
||||
'time': 0.4444},
|
||||
'test.five': {'name': 'test.five',
|
||||
'output': 'test.five',
|
||||
'status': 'OK',
|
||||
'time': 0.555},
|
||||
'test.six': {'name': 'test.six',
|
||||
'output': 'test.six',
|
||||
'status': 'OK',
|
||||
'time': 0.666}
|
||||
}
|
||||
|
||||
diff_ = diff.Diff(results1, results2, 0)
|
||||
assert len(diff_.diffs) == 8
|
||||
assert diff_.to_csv() != ''
|
||||
assert diff_.to_html() != ''
|
||||
assert diff_.to_json() != ''
|
||||
@ -31,6 +31,7 @@ _rally()
|
||||
OPTS["show_keypairs"]="--deploy-id"
|
||||
OPTS["show_networks"]="--deploy-id"
|
||||
OPTS["show_secgroups"]="--deploy-id"
|
||||
OPTS["verify_compare"]="--uuid-1 --uuid-2 --csv --html --json --output-file --threshold"
|
||||
OPTS["verify_detailed"]="--uuid --sort-by"
|
||||
OPTS["verify_list"]=""
|
||||
OPTS["verify_results"]="--uuid --html --json --output-file"
|
||||
@ -74,4 +75,4 @@ _rally()
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
complete -F _rally rally
|
||||
complete -F _rally rally
|
||||
|
||||
Loading…
Reference in New Issue
Block a user