Add benchmark overview page to html report

The overview page shows all scenarios and their summarized results
in single table, so it is easier to understand the whole result
and compare specific scenarios.

Also, there are some minor features and fixes in html report.

Changes:
  * Benchmark overview page
  * Task input file is available now
  * Scenario full duration is added to overview
  * Scenario duration is renamed to `Load duration'
  * Overview is sortable
  * Fix: control from browser history: back/forward buttons now work
  * Fix: durations are reset for iterations with errors
  * AngularJS version is updated to release 1.3.3
  * task sla_check output: column `success' with options True/False
    is renamed to `status' with options PASS/FAIL

Change-Id: I0eb7af01432c9c10e4ec55720bb53417478a5789
This commit is contained in:
Alexander Maretskiy 2014-11-21 19:27:57 +02:00
parent 9a06c00407
commit 867c4be47c
10 changed files with 553 additions and 280 deletions
rally
benchmark
cmd/commands
ui/templates
tests
ci/rally-gate
functional
unit
benchmark
cmd/commands

View File

@ -224,20 +224,19 @@ class BenchmarkEngine(object):
target=self.consume_results,
args=(key, self.task, runner.result_queue, is_done))
consumer.start()
context_obj = self._prepare_context(kw.get("context", {}),
name, self.admin)
# NOTE(boris-42): reset duration, in case of failures during
# context creation
self.duration = 0
self.full_duration = 0
try:
with base_ctx.ContextManager(context_obj):
self.duration = runner.run(name, context_obj,
kw.get("args", {}))
with rutils.Timer() as timer:
with base_ctx.ContextManager(context_obj):
self.duration = runner.run(name, context_obj,
kw.get("args", {}))
except Exception as e:
LOG.exception(e)
finally:
self.full_duration = timer.duration()
is_done.set()
consumer.join()
self.task.update_status(consts.TaskStatus.FINISHED)
@ -264,7 +263,8 @@ class BenchmarkEngine(object):
else:
time.sleep(0.1)
sla = base_sla.SLA.check_all(key['kw'], results)
sla = base_sla.SLA.check_all(key["kw"], results)
task.append_results(key, {"raw": results,
"scenario_duration": self.duration,
"load_duration": self.duration,
"full_duration": self.full_duration,
"sla": sla})

View File

@ -66,6 +66,10 @@ def _prepare_data(data):
"message": message,
"traceback": traceback})
# NOTE(maretskiy): Reset failed durations (no sense to display)
r["duration"] = 0
r["idle_duration"] = 0
durations.append(r["duration"])
idle_durations.append(r["idle_duration"])
@ -90,7 +94,8 @@ def _prepare_data(data):
"output_errors": output_errors,
"errors": errors,
"sla": data["sla"],
"duration": data["duration"],
"load_duration": data["load_duration"],
"full_duration": data["full_duration"],
}
@ -218,7 +223,7 @@ def _process_atomic(result, data):
def _get_atomic_action_durations(result):
raw = result.get('result', [])
raw = result.get("result", [])
actions_data = utils.get_atomic_actions_data(raw)
table = []
total = []
@ -248,8 +253,31 @@ def _get_atomic_action_durations(result):
return table
def _task_json(source_dict):
"""Generate task input file in JSON format.
:param source_dict: dict with input task data, in format:
{
scenario_name: [
{scenario config},
...
],
...
}
:returns: str JSON, ready for usage as task input file data
"""
source_list = []
indent = 2
for name, conf in sorted(source_dict.items()):
conf_str = '"%s": %s' % (name, json.dumps(conf, indent=indent))
source_list.append("\n".join(["%s%s" % (" " * indent, line)
for line in conf_str.split("\n")]))
return "{\n%s\n}" % ",\n".join(source_list)
def _process_results(results):
output = []
source_dict = {}
for result in results:
table_cols = ["Action",
"Min (sec)",
@ -260,32 +288,44 @@ def _process_results(results):
"Success",
"Count"]
table_rows = _get_atomic_action_durations(result)
name, kw, pos = (result["key"]["name"],
result["key"]["kw"], result["key"]["pos"])
scenario_name, kw, pos = (result["key"]["name"],
result["key"]["kw"], result["key"]["pos"])
data = _prepare_data(result)
cls = name.split(".")[0]
met = name.split(".")[1]
cls = scenario_name.split(".")[0]
met = scenario_name.split(".")[1]
name = "%s%s" % (met, (pos and " [%d]" % (int(pos) + 1) or ""))
try:
source_dict[scenario_name].append(kw)
except KeyError:
source_dict[scenario_name] = [kw]
output.append({
"cls": cls,
"met": met,
"pos": int(pos),
"name": "%s%s" % (met, (pos and " [%d]" % (int(pos) + 1) or "")),
"config": json.dumps({name: kw}, indent=2),
"duration": _process_main_duration(result, data),
"name": name,
"runner": kw["runner"]["type"],
"config": json.dumps({scenario_name: kw}, indent=2),
"iterations": _process_main_duration(result, data),
"atomic": _process_atomic(result, data),
"table_cols": table_cols,
"table_rows": table_rows,
"output": data["output"],
"output_errors": data["output_errors"],
"errors": data["errors"],
"total_duration": data["duration"],
"load_duration": data["load_duration"],
"full_duration": data["full_duration"],
"sla": data["sla"],
"sla_success": all([sla["success"] for sla in data["sla"]]),
"iterations_num": len(result["result"]),
})
return sorted(output, key=lambda r: "%s%s" % (r["cls"], r["name"]))
source = _task_json(source_dict)
scenarios = sorted(output, key=lambda r: "%s%s" % (r["cls"], r["name"]))
return source, scenarios
def plot(results):
data = _process_results(results)
template = ui_utils.get_template("task/report.mako")
return template.render(data=json.dumps(data))
source, scenarios = _process_results(results)
return template.render(data=json.dumps(scenarios),
source=json.dumps(source))

View File

@ -223,7 +223,7 @@ class TaskCommands(object):
print("args values:")
pprint.pprint(key["kw"])
scenario_time = result["data"]["scenario_duration"]
scenario_time = result["data"]["load_duration"]
raw = result["data"]["raw"]
table_cols = ["action", "min (sec)", "avg (sec)", "max (sec)",
"90 percentile", "95 percentile", "success",
@ -359,7 +359,8 @@ class TaskCommands(object):
results = map(lambda x: {"key": x["key"],
"sla": x["data"]["sla"],
"result": x["data"]["raw"],
"duration": x["data"]["scenario_duration"]},
"load_duration": x["data"]["load_duration"],
"full_duration": x["data"]["full_duration"]},
task.Task.get(task_id).get_results())
if out:
out = os.path.expanduser(out)
@ -415,16 +416,20 @@ class TaskCommands(object):
results = task.Task.get(task_id).get_results()
failed_criteria = 0
data = []
STATUS_PASS = "PASS"
STATUS_FAIL = "FAIL"
for result in results:
key = result["key"]
for sla in result["data"]["sla"]:
success = sla.pop("success")
sla["status"] = success and STATUS_PASS or STATUS_FAIL
sla["benchmark"] = key["name"]
sla["pos"] = key["pos"]
failed_criteria += 0 if sla['success'] else 1
failed_criteria += int(not success)
data.append(sla if tojson else rutils.Struct(**sla))
if tojson:
print(json.dumps(data))
else:
common_cliutils.print_list(data, ("benchmark", "pos", "criterion",
"success", "detail"))
"status", "detail"))
return failed_criteria

View File

@ -9,14 +9,19 @@
<style>
body { margin:0 0 50px; padding:0; font-size:14px; font-family:Helvetica,Arial,sans-serif }
a, a:active, a:focus, a:visited { text-decoration:none; outline:none }
h1 { color:#666; margin:0 0 25px; font-size:30px; font-weight:normal }
h2 { color:#666; margin:30px 0 15px; font-size:26px; font-weight:normal }
p { margin:5px 0; padding:15px 0 0 }
h1 { color:#666; margin:0 0 20px; font-size:30px; font-weight:normal }
h2 { color:#666; margin:25px 0 20px; font-size:26px; font-weight:normal }
table { border-collapse:collapse; border-spacing:0; width:100%; font-size:12px }
table th { text-align:left; padding:8px; color:#000; border:2px solid #ddd; border-width:0 0 2px 0 }
table th.sortable { cursor:pointer }
table td { text-align:left; border-top:1px solid #ddd; padding:8px; color:#333 }
table.compact td { padding:4px 8px }
table.striped tr:nth-child(odd) td { background:#f9f9f9 }
table.linked tbody tr:hover { background:#f9f9f9; cursor:pointer }
.richcolor td { color:#036; font-weight:bold }
.rich, .rich td { font-weight:bold }
.code { padding:10px; font-size:13px; color:#333; background:#f6f6f6; border:1px solid #e5e5e5; border-radius:4px }
.header { text-align:left; background:#333; font-size:18px; padding:13px 0; margin-bottom:20px; color:#fff; background-image:linear-gradient(to bottom, #444 0px, #222 100%) }
.header a, .header a:visited, .header a:focus { color:#999 }

View File

@ -7,20 +7,94 @@
<%block name="libs">
<link rel="stylesheet" href="http://cdnjs.cloudflare.com/ajax/libs/nvd3/1.1.15-beta/nv.d3.min.css">
<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/angularjs/1.3.0-rc.5/angular.min.js"></script>
<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/angularjs/1.3.3/angular.min.js"></script>
<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/d3/3.4.13/d3.min.js"></script>
<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/nvd3/1.1.15-beta/nv.d3.min.js"></script>
</%block>
<%block name="js_before">
"use strict";
if (typeof angular === "object") { angular.module("BenchmarkApp", []).controller(
"BenchmarkController", ["$scope", "$location", function($scope, $location) {
var app = angular.module("BenchmarkApp", [])
$scope.location = {
/* This is a junior brother of angular's $location, that allows non-`#'
symbol in uri, like `#/path/hash' instead of `#/path#hash' */
_splitter: "/",
normalize: function(str) {
/* Remove unwanted characters from string */
if (typeof str !== "string") { return "" }
return str.replace(/[^\w\-\.]/g, "")
},
_parseUri: function(uriStr) {
/* :returns: {path:string, hash:string} */
var self = this;
var obj = {path: "", hash: ""};
angular.forEach(uriStr.split(self._splitter), function(v){
var s = self.normalize(v);
if (! s) { return }
if (! this.path) { this.path = s } else if (! this.hash) { this.hash = s }
}, obj)
return obj
},
uri: function(obj) {
/* Getter/Setter */
if (! obj) { return this._parseUri($location.url()) }
if (obj.path && obj.hash) {
$location.url(obj.path + this._splitter + obj.hash)
} else if (obj.path) {
$location.url(obj.path)
} else {
$location.url("/")
}
},
path: function(path, hash) {
/* Getter/Setter */
var uri = this.uri();
if (path === "") { return this.uri({}) }
path = this.normalize(path);
if (! path) { return uri.path }
uri.path = path;
var _hash = this.normalize(hash);
if (_hash || hash === "") { uri.hash = _hash }
return this.uri(uri)
},
hash: function(hash) {
/* Getter/Setter */
var uri = this.uri();
if (! hash) { return uri.hash }
return this.uri({path:uri.path, hash:hash})
}
}
app.controller("BenchmarkController", ["$scope", "$location", function($scope, $location) {
/* Dispatch */
$scope.route = function(uri) {
if (! $scope.scenarios) {
return
}
if (uri.path in $scope.scenarios_map) {
$scope.view = {is_scenario:true};
$scope.scenario = $scope.scenarios_map[uri.path];
$scope.nav_idx = $scope.nav_map[uri.path];
$scope.showTab(uri.hash);
} else {
$scope.scenario = undefined
if (uri.path === "source") {
$scope.view = {is_source:true}
} else {
$scope.view = {is_main:true}
}
}
}
$scope.$on("$locationChangeSuccess", function (event, newUrl, oldUrl) {
$scope.route($scope.location.uri())
});
/* Navigation */
$scope.showNav = function(nav_idx) {
$scope.showNav = function(nav_idx) {
$scope.nav_idx = nav_idx
}
@ -30,7 +104,7 @@
{
id: "overview",
name: "Overview",
visible: function(){ return !! $scope.scenario.duration.pie.length }
visible: function(){ return !! $scope.scenario.iterations.pie.length }
},{
id: "details",
name: "Details",
@ -44,21 +118,22 @@
name: "Failures",
visible: function(){ return !! $scope.scenario.errors.length }
},{
id: "config",
name: "Config",
id: "task",
name: "Input task",
visible: function(){ return !! $scope.scenario.config }
}
];
$scope.tabs_map = {};
angular.forEach($scope.tabs,
function(tab){ this[tab.id] = tab }, $scope.tabs_map);
$scope.tabId = "overview";
$scope.showTab = function(tab_id) {
$scope.tab = tab_id in $scope.tabs_map ? tab_id : "overview"
}
for (var i in $scope.tabs) {
if ($scope.tabs[i].id === $location.hash()) {
$scope.tabId = $scope.tabs[i].id
}
$scope.tabs[i].showContent = function(){
$location.hash(this.id);
$scope.tabId = this.id
if ($scope.tabs[i].id === $scope.location.hash()) {
$scope.tab = $scope.tabs[i].id
}
$scope.tabs[i].isVisible = function(){
if ($scope.scenario) {
@ -66,11 +141,11 @@
return true
}
/* If tab should be hidden but is selected - show another one */
if (this.id === $location.hash()) {
if (this.id === $scope.location.hash()) {
for (var i in $scope.tabs) {
var tab = $scope.tabs[i];
if (tab.id != this.id && tab.visible()) {
tab.showContent();
$scope.tab = tab.id;
return false
}
}
@ -113,10 +188,10 @@
chart.xAxis
.axisLabel("Iteration (order number of method's call)")
.showMaxMin(false)
.tickFormat(d3.format('d'));
.tickFormat(d3.format("d"));
chart.yAxis
.axisLabel("Duration (seconds)")
.tickFormat(d3.format(',.2f'));
.tickFormat(d3.format(",.2f"));
this._render(selector, datum, chart)
},
histogram: function(selector, datum){
@ -129,10 +204,10 @@
.radioButtonMode(true)
chart.xAxis
.axisLabel("Duration (seconds)")
.tickFormat(d3.format(',.2f'));
.tickFormat(d3.format(",.2f"));
chart.yAxis
.axisLabel("Iterations (frequency)")
.tickFormat(d3.format('d'));
.tickFormat(d3.format("d"));
this._render(selector, datum, chart)
}
};
@ -141,13 +216,13 @@
if (! $scope.scenario) {
return
}
Charts.stack("#total-stack", $scope.scenario.duration.iter);
Charts.pie("#total-pie", $scope.scenario.duration.pie);
Charts.stack("#total-stack", $scope.scenario.iterations.iter);
Charts.pie("#total-pie", $scope.scenario.iterations.pie);
if ($scope.scenario.duration.histogram.length) {
if ($scope.scenario.iterations.histogram.length) {
var idx = this.totalHistogramModel.value;
Charts.histogram("#total-histogram",
[$scope.scenario.duration.histogram[idx]])
[$scope.scenario.iterations.histogram[idx]])
}
}
@ -173,28 +248,31 @@
}
}
/* Scenario */
$scope.showScenario = function(nav_idx, scenario_idx) {
$scope.nav_idx = nav_idx;
$scope.scenario_idx = scenario_idx;
$scope.scenario = $scope.scenarios[scenario_idx];
$location.path($scope.scenario.ref);
$scope.showError = function(message) {
return (function (e) {
e.style.display = "block";
e.textContent = message
})(document.getElementById("page-error"))
}
/* Initialization */
angular.element(document).ready(function () {
angular.element(document).ready(function(){
$scope.source = ${source};
$scope.scenarios = ${data};
if (! $scope.scenarios.length) {
return $scope.showError("Benchmark has empty scenarios data")
}
$scope.histogramOptions = [];
$scope.totalHistogramModel = {label:'', value:0};
$scope.atomicHistogramModel = {label:'', value:0};
/* Compose nav data */
/* Compose data mapping */
$scope.nav = [];
var nav_idx = 0;
var scenario_idx = 0;
$scope.nav_map = {};
$scope.scenarios_map = {};
var scenario_ref = $scope.location.path();
var met = [];
var itr = 0;
var cls_idx = 0;
@ -217,22 +295,24 @@
itr = 1
}
sc.ref = "/"+prev_cls+"."+sc.met+(itr > 1 ? "-"+itr : "");
if (sc.ref === $location.path()) {
scenario_idx = idx;
nav_idx = cls_idx;
sc.ref = $scope.location.normalize(sc.cls+"."+sc.met+(itr > 1 ? "-"+itr : ""));
$scope.scenarios_map[sc.ref] = sc;
$scope.nav_map[sc.ref] = cls_idx;
var current_ref = $scope.location.path();
if (sc.ref === current_ref) {
scenario_ref = sc.ref
}
met.push({name:sc.name, itr:itr, idx:idx});
met.push({name:sc.name, itr:itr, idx:idx, ref:sc.ref});
prev_met = sc.met;
itr += 1
/* Compose histograms options, from first suitable scenario */
if (! $scope.histogramOptions.length && sc.duration.histogram) {
for (var i in sc.duration.histogram) {
if (! $scope.histogramOptions.length && sc.iterations.histogram) {
for (var i in sc.iterations.histogram) {
$scope.histogramOptions.push({
label: sc.duration.histogram[i].method,
label: sc.iterations.histogram[i].method,
value: i
})
}
@ -247,24 +327,27 @@
/* Start */
$scope.showScenario(nav_idx, scenario_idx);
var uri = $scope.location.uri();
uri.path = scenario_ref;
$scope.route(uri);
$scope.$digest()
});
}])
}])}
</%block>
<%block name="css">
pre { padding:10px; font-size:13px; color:#333; background:#f5f5f5; border:1px solid #ccc; border-radius:4px }
.aside { margin:0 20px 0 0; display:block; width:255px; float:left }
.aside div:first-child { border-radius:4px 4px 0 0 }
.aside div:last-child { border-radius:0 0 4px 4px }
.aside > div { margin-bottom: 15px }
.aside > div div:first-child { border-top-left-radius:4px; border-top-right-radius:4px }
.aside > div div:last-child { border-bottom-left-radius:4px; border-bottom-right-radius:4px }
.nav-group { color:#678; background:#eee; border:1px solid #ddd; margin-bottom:-1px; display:block; padding:8px 9px; font-weight:bold; text-aligh:left; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; cursor:pointer }
.nav-group.active { color:#469 }
.nav-group.expanded { color:#469 }
.nav-group.active { background:#428bca; background-image:linear-gradient(to bottom, #428bca 0px, #3278b3 100%); border-color:#3278b3; color:#fff }
.nav-item { color:#555; background:#fff; border:1px solid #ddd; font-size:12px; display:block; margin-bottom:-1px; padding:8px 10px; text-aligh:left; text-overflow:ellipsis; white-space:nowrap; overflow:hidden; cursor:pointer }
.nav-item:hover { background:#f8f8f8 }
.nav-item.active, .nav-item.active:hover { background:#428bca; background-image:linear-gradient(to bottom, #428bca 0px, #3278b3 100%); border-color:#3278b3; color:#fff }
.tabs { list-style:outside none none; margin-bottom:0; padding-left:0; border-bottom:1px solid #ddd }
.tabs { list-style:outside none none; margin:0 0 5px; padding:0; border-bottom:1px solid #ddd }
.tabs:after { clear:both }
.tabs li { float:left; margin-bottom:-1px; display:block; position:relative }
.tabs li div { border:1px solid transparent; border-radius:4px 4px 0 0; line-height:20px; margin-right:2px; padding:10px 15px; color:#428bca }
@ -281,7 +364,7 @@
.expandable { cursor:pointer }
.clearfix { clear:both }
.top-margin { margin-top:40px !important }
.sortable > .arrow { display:inline-block; width:12px; height:inherit; color:#c90 }
.content-main { margin:0 5px; display:block; float:left }
</%block>
@ -300,169 +383,274 @@
<%block name="content">
<p id="page-error" class="notify-error" style="display:none"></p>
<div id="scenarios-list" class="aside" ng-show="scenario" ng-cloack>
<div class="nav-group" title="{{n.cls}}"
ng-repeat-start="n in nav track by $index"
ng-click="showNav(n.idx)"
ng-class="{active:n.idx == nav_idx}">
<span ng-hide="n.idx == nav_idx">&#9658;</span>
<span ng-show="n.idx == nav_idx">&#9660;</span>
{{n.cls}}</div>
<div class="nav-item" title="{{m.name}}"
ng-show="n.idx == nav_idx"
ng-class="{active:m.idx == scenario_idx}"
ng-click="showScenario(n.idx, m.idx)"
ng-repeat="m in n.met track by $index"
ng-repeat-end>{{m.name}}</div>
<div id="content-nav" class="aside" ng-show="scenarios.length" ng-cloack>
<div>
<div class="nav-group"
ng-class="{active:view.is_main}"
ng-click="location.path('')">Benchmark overview</div>
<div class="nav-group"
ng-class="{active:view.is_source}"
ng-click="location.path('source', '')">Input file</div>
</div>
<div>
<div class="nav-group" title="{{n.cls}}"
ng-repeat-start="n in nav track by $index"
ng-click="showNav(n.idx)"
ng-class="{expanded:n.idx==nav_idx}">
<span ng-hide="n.idx==nav_idx">&#9658;</span>
<span ng-show="n.idx==nav_idx">&#9660;</span>
{{n.cls}}</div>
<div class="nav-item" title="{{m.name}}"
ng-show="n.idx==nav_idx"
ng-class="{active:m.ref==scenario.ref}"
ng-click="location.path(m.ref)"
ng-repeat="m in n.met track by $index"
ng-repeat-end>{{m.name}}</div>
</div>
</div>
<div id="scenario-data" class="content-main" ng-show="scenario" ng-cloak>
<div id="content-main" class="content-main" ng-show="scenarios.length" ng-cloak>
<h1>{{scenario.cls}}.<wbr>{{scenario.name}} ({{scenario.total_duration | number:2}}s)</h1>
<ul class="tabs">
<li ng-repeat="tab in tabs"
ng-class="{active:tab.id == tabId}"
ng-click="tab.showContent()"
ng-show="tab.isVisible()">
<div>{{tab.name}}</div>
</li>
<div class="clearfix"></div>
</ul>
<div ng-include="tabId"></div>
<div ng-show="view.is_main">
<h1>Benchmark overview</h1>
<table class="linked compact"
ng-init="ov_srt='ref'; ov_dir=false">
<thead>
<tr>
<th class="sortable"
title="Scenario name, with optional suffix of call number"
ng-click="ov_srt='ref'; ov_dir=!ov_dir">
Scenario
<span class="arrow">
<b ng-show="ov_srt=='ref' && !ov_dir">&#x25b4;</b>
<b ng-show="ov_srt=='ref' && ov_dir">&#x25be;</b>
</span>
<th class="sortable"
title="How long the scenario run, without context duration"
ng-click="ov_srt='load_duration'; ov_dir=!ov_dir">
Load duration (s)
<span class="arrow">
<b ng-show="ov_srt=='load_duration' && !ov_dir">&#x25b4;</b>
<b ng-show="ov_srt=='load_duration' && ov_dir">&#x25be;</b>
</span>
<th class="sortable"
title="Scenario duration plus context duration"
ng-click="ov_srt='full_duration'; ov_dir=!ov_dir">
Full duration (s)
<span class="arrow">
<b ng-show="ov_srt=='full_duration' && !ov_dir">&#x25b4;</b>
<b ng-show="ov_srt=='full_duration' && ov_dir">&#x25be;</b>
</span>
<th class="sortable"
title="Number of iterations"
ng-click="ov_srt='iterations_num'; ov_dir=!ov_dir">
Iterations
<span class="arrow">
<b ng-show="ov_srt=='iterations_num' && !ov_dir">&#x25b4;</b>
<b ng-show="ov_srt=='iterations_num' && ov_dir">&#x25be;</b>
</span>
<th class="sortable"
title="Scenario runner type"
ng-click="ov_srt='runner'; ov_dir=!ov_dir">
Runner
<span class="arrow">
<b ng-show="ov_srt=='runner' && !ov_dir">&#x25b4;</b>
<b ng-show="ov_srt=='runner' && ov_dir">&#x25be;</b>
</span>
<th class="sortable"
title="Number of errors occured"
ng-click="ov_srt='errors.length'; ov_dir=!ov_dir">
Errors
<span class="arrow">
<b ng-show="ov_srt=='errors.length' && !ov_dir">&#x25b4;</b>
<b ng-show="ov_srt=='errors.length' && ov_dir">&#x25be;</b>
</span>
<th class="sortable"
title="Whether SLA check is successful"
ng-click="ov_srt='sla_success'; ov_dir=!ov_dir">
Success (SLA)
<span class="arrow">
<b ng-show="ov_srt=='sla_success' && !ov_dir">&#x25b4;</b>
<b ng-show="ov_srt=='sla_success' && ov_dir">&#x25be;</b>
</span>
<tr>
</thead>
<tbody>
<tr ng-repeat="sc in scenarios | orderBy:ov_srt:ov_dir"
ng-click="location.path(sc.ref)">
<td>{{sc.ref}}
<td>{{sc.load_duration | number:3}}
<td>{{sc.full_duration | number:3}}
<td>{{sc.iterations_num}}
<td>{{sc.runner}}
<td>{{sc.errors.length}}
<td>
<span ng-show="sc.sla_success" class="status-pass">&#x2714;</span>
<span ng-hide="sc.sla_success" class="status-fail">&#x2716;</span>
<tr>
</tbody>
</table>
</div>
<script type="text/ng-template" id="overview">
{{renderTotal()}}
<div ng-show="view.is_source">
<h1>Input file</h1>
<pre class="code">{{source}}</pre>
</div>
<div ng-show="scenario.sla.length">
<h2>Service-level agreement</h2>
<div ng-show="view.is_scenario">
<h1>{{scenario.cls}}.<wbr>{{scenario.name}} ({{scenario.full_duration | number:3}}s)</h1>
<ul class="tabs">
<li ng-repeat="t in tabs"
ng-show="t.isVisible()"
ng-class="{active:t.id == tab}"
ng-click="location.hash(t.id)">
<div>{{t.name}}</div>
</li>
<div class="clearfix"></div>
</ul>
<div ng-include="tab"></div>
<script type="text/ng-template" id="overview">
{{renderTotal()}}
<p>
Load duration: <b>{{scenario.load_duration | number:3}} s</b> &nbsp;
Full duration: <b>{{scenario.full_duration | number:3}} s</b> &nbsp;
Iterations: <b>{{scenario.iterations_num}}</b> &nbsp;
Failures: <b>{{scenario.errors.length}}</b>
</p>
<div ng-show="scenario.sla.length">
<h2>Service-level agreement</h2>
<table class="striped">
<thead>
<tr>
<th>Criterion
<th>Detail
<th>Success
<tr>
</thead>
<tbody>
<tr class="rich"
ng-repeat="row in scenario.sla track by $index"
ng-class="{'status-fail':!row.success, 'status-pass':row.success}">
<td>{{row.criterion}}
<td>{{row.detail}}
<td class="capitalize">{{row.success}}
<tr>
</tbody>
</table>
</div>
<h2>Total durations</h2>
<table class="striped">
<thead>
<tr>
<th>Criterion
<th>Detail
<th>Success
<th ng-repeat="i in scenario.table_cols track by $index">{{i}}
<tr>
</thead>
<tbody>
<tr class="rich"
ng-repeat="row in scenario.sla track by $index"
ng-class="{'status-fail':!row.success, 'status-pass':row.success}">
<td>{{row.criterion}}
<td>{{row.detail}}
<td class="capitalize">{{row.success}}
<tr ng-class="{richcolor:$last}"
ng-repeat="row in scenario.table_rows track by $index">
<td ng-repeat="i in row track by $index">{{i}}
<tr>
</tbody>
</table>
</div>
<h2>Total durations</h2>
<table class="striped">
<thead>
<tr>
<th ng-repeat="i in scenario.table_cols track by $index">{{i}}
<tr>
</thead>
<tbody>
<tr ng-class="{richcolor:$last}" ng-repeat="row in scenario.table_rows track by $index">
<td ng-repeat="i in row track by $index">{{i}}
<tr>
</tbody>
</table>
<h2>Charts for the Total durations</h2>
<div class="chart">
<svg id="total-stack"></svg>
</div>
<h2>Charts for the Total durations</h2>
<div class="chart">
<svg id="total-stack"></svg>
</div>
<div class="chart lesser top-margin">
<svg id="total-pie"></svg>
</div>
<div class="chart lesser top-margin">
<svg id="total-pie"></svg>
</div>
<div class="chart larger top-margin"
ng-show="scenario.iterations.histogram.length">
<svg id="total-histogram"></svg>
<select class="chart-dropdown"
ng-model="totalHistogramModel"
ng-options="i.label for i in histogramOptions"></select>
</div>
</script>
<div class="chart larger top-margin"
ng-show="scenario.duration.histogram.length">
<svg id="total-histogram"></svg>
<select class="chart-dropdown"
ng-model="totalHistogramModel"
ng-options="i.label for i in histogramOptions"></select>
</div>
</script>
<script type="text/ng-template" id="details">
{{renderDetails()}}
<script type="text/ng-template" id="details">
{{renderDetails()}}
<h2>Charts for each Atomic Action</h2>
<div class="chart">
<svg id="atomic-stack"></svg>
</div>
<h2>Charts for each Atomic Action</h2>
<div class="chart">
<svg id="atomic-stack"></svg>
</div>
<div class="chart lesser top-margin">
<svg id="atomic-pie"></svg>
</div>
<div class="chart lesser top-margin">
<svg id="atomic-pie"></svg>
</div>
<div class="chart larger top-margin"
ng-show="scenario.atomic.histogram.length">
<svg id="atomic-histogram"></svg>
<select class="chart-dropdown"
ng-model="atomicHistogramModel"
ng-options="i.label for i in histogramOptions"></select>
</div>
</script>
<div class="chart larger top-margin"
ng-show="scenario.atomic.histogram.length">
<svg id="atomic-histogram"></svg>
<select class="chart-dropdown"
ng-model="atomicHistogramModel"
ng-options="i.label for i in histogramOptions"></select>
</div>
</script>
<script type="text/ng-template" id="output">
{{renderOutput()}}
<script type="text/ng-template" id="output">
{{renderOutput()}}
<h2>Scenario output</h2>
<div class="chart">
<svg id="output-stack"></svg>
</div>
</script>
<h2>Scenario output</h2>
<div class="chart">
<svg id="output-stack"></svg>
</div>
</script>
<script type="text/ng-template" id="failures">
<h2>Benchmark failures (<ng-pluralize
count="scenario.errors.length"
when="{'1': '1 iteration', 'other': '{} iterations'}"></ng-pluralize> failed)
</h2>
<table class="striped">
<thead>
<tr>
<th>
<th>Iteration
<th>Exception type
<th>Exception message
</tr>
</thead>
<tbody>
<tr class="expandable"
ng-repeat-start="i in scenario.errors track by $index"
ng-click="i.expanded = ! i.expanded">
<td>
<span ng-hide="i.expanded">&#9658;</span>
<span ng-show="i.expanded">&#9660;</span>
<td>{{i.iteration}}
<td>{{i.type}}
<td class="failure-mesg">{{i.message}}
</tr>
<tr ng-show="i.expanded" ng-repeat-end>
<td colspan="4" class="failure-trace">{{i.traceback}}
</tr>
</tbody>
</table>
</script>
<script type="text/ng-template" id="failures">
<h2>Benchmark failures (<ng-pluralize
count="scenario.errors.length"
when="{'1': '1 iteration', 'other': '{} iterations'}"></ng-pluralize> failed)
</h2>
<table class="striped">
<thead>
<tr>
<th>
<th>Iteration
<th>Exception type
<th>Exception message
</tr>
</thead>
<tbody>
<tr class="expandable"
ng-repeat-start="i in scenario.errors track by $index"
ng-click="i.expanded = ! i.expanded">
<td>
<span ng-hide="i.expanded">&#9658;</span>
<span ng-show="i.expanded">&#9660;</span>
<td>{{i.iteration}}
<td>{{i.type}}
<td class="failure-mesg">{{i.message}}
</tr>
<tr ng-show="i.expanded" ng-repeat-end>
<td colspan="4" class="failure-trace">{{i.traceback}}
</tr>
</tbody>
</table>
</script>
<script type="text/ng-template" id="config">
<h2>Scenario Configuration</h2>
<pre>{{scenario.config}}</pre>
</script>
<script type="text/ng-template" id="task">
<h2>Scenario Configuration</h2>
<pre class="code">{{scenario.config}}</pre>
</script>
</div>
</div>
<div class="clearfix"></div>
</%block>
<%block name="js_after">
if (! window.angular) {
document.getElementById("page-error").style.display = "block";
document.getElementById("page-error").textContent = "Failed to load AngularJS framework";
document.getElementById("scenarios-list").style.display = "none";
document.getElementById("scenario-data").style.display = "none";
}
if (! window.angular) {(function(f){
f(document.getElementById("content-nav"), "none");
f(document.getElementById("content-main"), "none");
f(document.getElementById("page-error"), "block").textContent = "Failed to load AngularJS framework"
})(function(e, s){e.style.display = s; return e})}
</%block>

View File

@ -26,8 +26,7 @@
<%block name="content">
<h2>Logs and files</h2>
<ul class="columns">
<li><a href="rally-plot/task.txt" class="rich">Rally input task</a>
<li><a href="console.html">Benchmarking logs</a> <code>console.html</code>
<li><a href="console.html" class="rich">Benchmarking logs</a> <code>console.html</code>
<li><a href="logs/">Logs of all services</a> <code>logs/</code>
<li><a href="rally-plot/">Rally files</a> <code>rally-plot/</code>
</ul>
@ -37,7 +36,7 @@
<li><a href="rally-plot/results.html.gz" class="rich">HTML report</a> <code>$ rally task report</code>
<li><a href="rally-plot/detailed.txt.gz">Text report</a> <code>$ rally task detailed</code>
<li><a href="rally-plot/detailed_with_iterations.txt.gz">Text report detailed</a> <code>$ rally task detailed --iterations-data</code>
<li><a href="rally-plot/sla.txt" class="rich">Success criteria (SLA)</a> <code>$ rally task sla_check</code>
<li><a href="rally-plot/sla.txt">Success criteria (SLA)</a> <code>$ rally task sla_check</code>
<li><a href="rally-plot/results.json.gz">Raw results (JSON)</a> <code>$ rally task results</code>
</ul>

View File

@ -125,11 +125,11 @@ class SLATestCase(unittest.TestCase):
{"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_seconds_per_iteration",
"detail": mock.ANY,
"pos": 0, "success": True},
"pos": 0, "status": "PASS"},
{"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_failure_percent",
"detail": mock.ANY,
"pos": 0, "success": True},
"pos": 0, "status": "PASS"},
]
data = rally("task sla_check --json", getjson=True)
self.assertEqual(expected, data)

View File

@ -20,35 +20,56 @@ import mock
from rally.benchmark.processing import plot
from tests.unit import test
PLOT = "rally.benchmark.processing.plot."
class PlotTestCase(test.TestCase):
@mock.patch("rally.benchmark.processing.plot.ui_utils")
@mock.patch("rally.benchmark.processing.plot._process_results")
@mock.patch(PLOT + "ui_utils")
@mock.patch(PLOT + "_process_results")
def test_plot(self, mock_proc_results, mock_utils):
mock_render = mock.Mock(return_value="plot_html")
mock_utils.get_template = mock.Mock(
return_value=mock.Mock(render=mock_render))
mock_proc_results.return_value = [{"name": "a"}, {"name": "b"}]
task_data = [{"name": "a"}, {"name": "b"}]
task_source = "JSON"
mock_proc_results.return_value = (task_source, task_data)
result = plot.plot(["abc"])
self.assertEqual(result, "plot_html")
mock_render.assert_called_once_with(
data=json.dumps(mock_proc_results.return_value)
data=json.dumps(task_data),
source=json.dumps(task_source)
)
mock_utils.get_template.assert_called_once_with("task/report.mako")
@mock.patch("rally.benchmark.processing.plot._prepare_data")
@mock.patch("rally.benchmark.processing.plot._process_atomic")
@mock.patch("rally.benchmark.processing.plot._process_main_duration")
def test__process_results(self, mock_main_duration, mock_atomic,
mock_prepare):
results = [
{"key": {"name": "Klass.method_foo", "pos": 0, "kw": "config1"}},
{"key": {"name": "Klass.method_foo", "pos": 1, "kw": "config2"}},
{"key": {"name": "Klass.method_bar", "pos": 0, "kw": "config3"}}
]
def test__task_json(self):
self.assertRaises(TypeError, plot._task_json)
self.assertRaises(AttributeError, plot._task_json, [])
self.assertEqual(plot._task_json({"foo": ["a", "b"]}),
'{\n "foo": [\n "a", \n "b"\n ]\n}')
self.assertEqual(plot._task_json({"foo": ["a", "b"], "bar": ["c"]}),
('{\n "bar": [\n "c"\n ],'
'\n "foo": [\n "a", \n "b"\n ]\n}'))
@mock.patch(PLOT + "_task_json")
@mock.patch(PLOT + "_prepare_data")
@mock.patch(PLOT + "_process_atomic")
@mock.patch(PLOT + "_get_atomic_action_durations")
@mock.patch(PLOT + "_process_main_duration")
def test__process_results(self, mock_main_duration, mock_get_atomic,
mock_atomic, mock_prepare, mock_task_json):
sla = [{"success": True}]
result = ["iter_1", "iter_2"]
iterations = len(result)
kw = {"runner": {"type": "foo_runner"}}
result_ = lambda i: {
"key": {"pos": i,
"name": "Class.method",
"kw": kw},
"result": result,
"sla": sla}
results = [result_(i) for i in 0, 1, 2]
table_cols = ["Action",
"Min (sec)",
"Avg (sec)",
@ -57,41 +78,51 @@ class PlotTestCase(test.TestCase):
"95 percentile",
"Success",
"Count"]
atomic_durations = [["atomic_1"], ["atomic_2"]]
mock_prepare.side_effect = lambda i: {"errors": "errors_list",
"output": [],
"output_errors": [],
"sla": "foo_sla",
"duration": 12345.67}
"sla": i["sla"],
"load_duration": 1234.5,
"full_duration": 6789.1}
mock_main_duration.return_value = "main_duration"
mock_get_atomic.return_value = atomic_durations
mock_atomic.return_value = "main_atomic"
mock_task_json.return_value = "JSON"
output = plot._process_results(results)
source, scenarios = plot._process_results(results)
source_dict = {"Class.method": [kw] * len(results)}
mock_task_json.assert_called_with(source_dict)
self.assertEqual(source, "JSON")
results = sorted(results, key=lambda r: "%s%s" % (r["key"]["name"],
r["key"]["pos"]))
for i, r in enumerate(results):
config = json.dumps({r["key"]["name"]: r["key"]["kw"]}, indent=2)
pos = int(r["key"]["pos"])
cls = r["key"]["name"].split(".")[0]
met = r["key"]["name"].split(".")[1]
name = "%s%s" % (met, (pos and " [%d]" % (pos + 1) or ""))
self.assertEqual(output[i], {
self.assertEqual(scenarios[i], {
"cls": cls,
"pos": r["key"]["pos"],
"met": met,
"name": name,
"config": config,
"duration": mock_main_duration.return_value,
"iterations": mock_main_duration.return_value,
"atomic": mock_atomic.return_value,
"table_cols": table_cols,
"table_rows": [["total", None, None, None, None, None, 0, 0]],
"table_rows": atomic_durations,
"errors": "errors_list",
"output": [],
"output_errors": [],
"sla": "foo_sla",
"total_duration": 12345.67
"runner": "foo_runner",
"sla": sla,
"sla_success": True,
"iterations_num": iterations,
"load_duration": 1234.5,
"full_duration": 6789.1
})
def test__process_main_time(self):
@ -120,7 +151,8 @@ class PlotTestCase(test.TestCase):
}
],
"sla": "foo_sla",
"duration": 12345.67
"load_duration": 1234.5,
"full_duration": 6789.1
}
output = plot._process_main_duration(result,
@ -134,11 +166,11 @@ class PlotTestCase(test.TestCase):
"iter": [
{
"key": "duration",
"values": [(1, 1.0), (2, 1.0), (3, 2.0)]
"values": [(1, 1.0), (2, 0), (3, 2.0)]
},
{
"key": "idle_duration",
"values": [(1, 2.0), (2, 1.0), (3, 3.0)]
"values": [(1, 2.0), (2, 0), (3, 3.0)]
}
],
"histogram": [
@ -281,7 +313,8 @@ class PlotTestCase(test.TestCase):
mock_compress.side_effect = lambda i, **kv: i
rows_num = 100
total_duration = 12345.67
load_duration = 1234.5
full_duration = 6789.1
sla = [{"foo": "bar"}]
data = []
for i in range(rows_num):
@ -305,10 +338,15 @@ class PlotTestCase(test.TestCase):
values_atomic_a1 = [i + 0.1 for i in range(rows_num)]
values_atomic_a2 = [i + 0.8 for i in range(rows_num)]
values_duration = [i * 3.1 for i in range(rows_num)]
values_duration[42] = 0
values_duration[52] = 0
values_idle = [i * 0.2 for i in range(rows_num)]
values_idle[42] = 0
values_idle[52] = 0
prepared_data = plot._prepare_data({"result": data,
"duration": total_duration,
"load_duration": load_duration,
"full_duration": full_duration,
"sla": sla,
"key": "foo_key"})
self.assertEqual(2, len(prepared_data["errors"]))
@ -338,6 +376,7 @@ class PlotTestCase(test.TestCase):
"type": "spam"}],
"output": expected_output,
"output_errors": expected_output_errors,
"duration": total_duration,
"load_duration": load_duration,
"full_duration": full_duration,
"sla": sla,
}, prepared_data)

View File

@ -319,6 +319,7 @@ class BenchmarkEngineTestCase(test.TestCase):
is_done = mock.MagicMock()
is_done.isSet.side_effect = [False, False, True]
eng = engine.BenchmarkEngine(config, task)
eng.duration = 1
eng.duration = 123
eng.full_duration = 456
eng.consume_results(key, task, collections.deque([1, 2]), is_done)
mock_check_all.assert_called_once_with({"fake": 2}, [1, 2])

View File

@ -110,7 +110,7 @@ class TaskCommandsTestCase(test.TestCase):
"kw": "fake_kw"
},
"data": {
"scenario_duration": 1.0,
"load_duration": 1.0,
"raw": []
}
}
@ -121,7 +121,7 @@ class TaskCommandsTestCase(test.TestCase):
self.task.detailed(test_uuid)
mock_db.task_get_detailed.assert_called_once_with(test_uuid)
@mock.patch('rally.cmd.commands.task.envutils.get_global')
@mock.patch("rally.cmd.commands.task.envutils.get_global")
def test_detailed_no_task_id(self, mock_default):
mock_default.side_effect = exceptions.InvalidArgumentsException
self.assertRaises(exceptions.InvalidArgumentsException,
@ -172,14 +172,16 @@ class TaskCommandsTestCase(test.TestCase):
task_id = "foo_task_id"
data = [
{"key": "foo_key", "data": {"raw": "foo_raw", "sla": "foo_sla",
"scenario_duration": "foo_duration"}},
"load_duration": 1.1,
"full_duration": 1.2}},
{"key": "bar_key", "data": {"raw": "bar_raw", "sla": "bar_sla",
"scenario_duration": "bar_duration"}},
]
"load_duration": 2.1,
"full_duration": 2.2}}]
results = map(lambda x: {"key": x["key"],
"result": x["data"]["raw"],
"sla": x["data"]["sla"],
"duration": x["data"]["scenario_duration"]},
"load_duration": x["data"]["load_duration"],
"full_duration": x["data"]["full_duration"]},
data)
mock_results = mock.Mock(return_value=data)
mock_get.return_value = mock.Mock(get_results=mock_results)
@ -257,33 +259,27 @@ class TaskCommandsTestCase(test.TestCase):
in task_uuids]
self.assertTrue(mock_api.delete_task.mock_calls == expected_calls)
@mock.patch('rally.cmd.commands.task.common_cliutils.print_list')
@mock.patch("rally.cmd.commands.task.common_cliutils.print_list")
@mock.patch("rally.cmd.commands.task.db")
def _test_sla_check(self, mock_db, mock_print_list):
value = [{
"key": {
"name": "fake_name",
"pos": "fake_pos",
"kw": "fake_kw"
},
"data": {
"scenario_duration": 1.0,
"raw": [],
"sla":
[{"benchmark": "KeystoneBasic.create_user",
"criterion": "max_seconds_per_iteration",
"pos": 0, "success": False, "detail":
"Maximum seconds per iteration 4s, actually 5s"}]
}
}]
mock_db.task_result_get_all_by_uuid.return_value = value
retval = self.task.sla_check(task_id='fake_task_id')
self.assertEqual(1, retval)
data = [{"key": {"name": "fake_name",
"pos": "fake_pos",
"kw": "fake_kw"},
"data": {"scenario_duration": 42.0,
"raw": [],
"sla": [{"benchmark": "KeystoneBasic.create_user",
"criterion": "max_seconds_per_iteration",
"pos": 0,
"success": False,
"detail": "Max foo, actually bar"}]}}]
mock_db.task_result_get_all_by_uuid.return_value = data
result = self.task.sla_check(task_id="fake_task_id")
self.assertEqual(1, result)
@mock.patch('rally.cmd.commands.task.open',
@mock.patch("rally.cmd.commands.task.open",
mock.mock_open(read_data='{"some": "json"}'),
create=True)
@mock.patch('rally.orchestrator.api.task_validate')
@mock.patch("rally.orchestrator.api.task_validate")
def test_verify(self, mock_validate):
self.task.validate('path_to_config.json', 'fake_id')
mock_validate.assert_called_once_with('fake_id', {"some": "json"})
self.task.validate("path_to_config.json", "fake_id")
mock_validate.assert_called_once_with("fake_id", {"some": "json"})