[Reports][CLI] Introduce Trends report
New HTML report for statistics trends for given tasks which is generated by cli command "rally task trends". Blueprint: trends-report Change-Id: I0805058e8bd225796b02516fad094d73037d5495
This commit is contained in:
Alexander Maretskiy
parent
164f310938
commit
b7fa1422f6
@ -43,6 +43,7 @@ _rally()
|
||||
OPTS["task_sla_check"]="--uuid --json"
|
||||
OPTS["task_start"]="--deployment --task --task-args --task-args-file --tag --no-use --abort-on-sla-failure"
|
||||
OPTS["task_status"]="--uuid"
|
||||
OPTS["task_trends"]="--out --open --tasks"
|
||||
OPTS["task_use"]="--uuid"
|
||||
OPTS["task_validate"]="--deployment --task --task-args --task-args-file"
|
||||
OPTS["verify_compare"]="--uuid-1 --uuid-2 --csv --html --json --output-file --threshold"
|
||||
|
||||
@ -543,6 +543,67 @@ class TaskCommands(object):
|
||||
print(_("There are no tasks. To run a new task, use:\n"
|
||||
"\trally task start"))
|
||||
|
||||
@cliutils.args("--out", metavar="<path>",
|
||||
type=str, dest="out", required=False,
|
||||
help="Path to output file.")
|
||||
@cliutils.args("--open", dest="open_it", action="store_true",
|
||||
help="Open the output in a browser.")
|
||||
@cliutils.args("--tasks", dest="tasks", nargs="+",
|
||||
help="UUIDs of tasks, or JSON files with task results")
|
||||
@cliutils.suppress_warnings
|
||||
def trends(self, *args, **kwargs):
|
||||
"""Generate workloads trends HTML report."""
|
||||
tasks = kwargs.get("tasks", []) or list(args)
|
||||
|
||||
if not tasks:
|
||||
print(_("ERROR: At least one task must be specified"),
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
|
||||
results = []
|
||||
for task_id in tasks:
|
||||
if os.path.exists(os.path.expanduser(task_id)):
|
||||
with open(os.path.expanduser(task_id), "r") as inp_js:
|
||||
task_results = json.load(inp_js)
|
||||
for result in task_results:
|
||||
try:
|
||||
jsonschema.validate(
|
||||
result,
|
||||
api.Task.TASK_RESULT_SCHEMA)
|
||||
except jsonschema.ValidationError as e:
|
||||
print(_("ERROR: Invalid task result format in %s")
|
||||
% task_id, file=sys.stderr)
|
||||
print(six.text_type(e), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
elif uuidutils.is_uuid_like(task_id):
|
||||
task_results = map(
|
||||
lambda x: {"key": x["key"],
|
||||
"sla": x["data"]["sla"],
|
||||
"result": x["data"]["raw"],
|
||||
"load_duration": x["data"]["load_duration"],
|
||||
"full_duration": x["data"]["full_duration"]},
|
||||
api.Task.get(task_id).get_results())
|
||||
else:
|
||||
print(_("ERROR: Invalid UUID or file name passed: %s")
|
||||
% task_id, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
results.extend(task_results)
|
||||
|
||||
result = plot.trends(results)
|
||||
|
||||
out = kwargs.get("out")
|
||||
if out:
|
||||
output_file = os.path.expanduser(out)
|
||||
|
||||
with open(output_file, "w+") as f:
|
||||
f.write(result)
|
||||
if kwargs.get("open_it"):
|
||||
webbrowser.open_new_tab("file://" + os.path.realpath(out))
|
||||
else:
|
||||
print(result)
|
||||
|
||||
@cliutils.args("--tasks", dest="tasks", nargs="+",
|
||||
help="UUIDs of tasks, or JSON files with task results")
|
||||
@cliutils.args("--out", metavar="<path>",
|
||||
|
||||
@ -154,6 +154,14 @@ def plot(tasks_results, include_libs=False):
|
||||
include_libs=include_libs)
|
||||
|
||||
|
||||
def trends(tasks_results):
|
||||
trends = Trends()
|
||||
for i, scenario in enumerate(_extend_results(tasks_results), 1):
|
||||
trends.add_result(scenario)
|
||||
template = ui_utils.get_template("task/trends.html")
|
||||
return template.render(data=json.dumps(trends.get_data()))
|
||||
|
||||
|
||||
class Trends(object):
|
||||
"""Process tasks results and make trends data.
|
||||
|
||||
@ -242,6 +250,7 @@ class Trends(object):
|
||||
("max", charts.streaming.MaxComputation()),
|
||||
("avg", charts.streaming.MeanComputation())):
|
||||
for k, v in total[stat]:
|
||||
if isinstance(v, (float,) + six.integer_types):
|
||||
comp.add(v)
|
||||
self._tasks[key]["stat"][stat] = comp.result()
|
||||
del self._tasks[key]["data"]
|
||||
|
||||
@ -317,6 +317,40 @@ class TaskTestCase(unittest.TestCase):
|
||||
self.assertTrue(os.path.exists(html_report))
|
||||
self._assert_html_report_libs_are_embedded(html_report)
|
||||
|
||||
def test_trends(self):
|
||||
cfg1 = {
|
||||
"Dummy.dummy": [
|
||||
{"runner": {"type": "constant", "times": 2,
|
||||
"concurrency": 2}}],
|
||||
"Dummy.dummy_random_action": [
|
||||
{"args": {"actions_num": 4},
|
||||
"runner": {"type": "constant", "times": 2, "concurrency": 2}},
|
||||
{"runner": {"type": "constant", "times": 2,
|
||||
"concurrency": 2}}]}
|
||||
cfg2 = {
|
||||
"Dummy.dummy": [
|
||||
{"args": {"sleep": 0.6},
|
||||
"runner": {"type": "constant", "times": 2,
|
||||
"concurrency": 2}}]}
|
||||
|
||||
config1 = utils.TaskConfig(cfg1)
|
||||
config2 = utils.TaskConfig(cfg2)
|
||||
rally = utils.Rally()
|
||||
report = rally.gen_report_path(extension="html")
|
||||
|
||||
for i in range(5):
|
||||
rally("task start --task %(file)s --tag trends_run_%(idx)d"
|
||||
% {"file": config1.filename, "idx": i})
|
||||
rally("task start --task %s --tag trends_run_once" % config2.filename)
|
||||
|
||||
tasks_list = rally("task list")
|
||||
uuids = [u[2:38] for u in tasks_list.split("\n") if "trends_run" in u]
|
||||
|
||||
rally("task trends %(uuids)s --out %(report)s"
|
||||
% {"uuids": " ".join(uuids), "report": report})
|
||||
del config1, config2
|
||||
self.assertTrue(os.path.exists(report))
|
||||
|
||||
def test_delete(self):
|
||||
rally = utils.Rally()
|
||||
cfg = self._get_sample_task_config()
|
||||
|
||||
@ -422,6 +422,120 @@ class TaskCommandsTestCase(test.TestCase):
|
||||
consts.TaskStatus.ABORTED)))
|
||||
mock_stdout.write.assert_has_calls([mock.call(expected_out)])
|
||||
|
||||
def _make_result(self, keys):
|
||||
return [{"key": {"name": key, "pos": 0},
|
||||
"data": {"raw": key + "_raw",
|
||||
"sla": key + "_sla",
|
||||
"load_duration": 1.2,
|
||||
"full_duration": 2.3}} for key in keys]
|
||||
|
||||
@mock.patch("rally.cli.commands.task.jsonschema.validate",
|
||||
return_value=None)
|
||||
@mock.patch("rally.cli.commands.task.os.path")
|
||||
@mock.patch("rally.cli.commands.task.open", create=True)
|
||||
@mock.patch("rally.cli.commands.task.plot")
|
||||
@mock.patch("rally.cli.commands.task.api.Task.get")
|
||||
@mock.patch("rally.cli.commands.task.webbrowser")
|
||||
def test_trends(self, mock_webbrowser, mock_task_get, mock_plot,
|
||||
mock_open, mock_os_path, mock_validate):
|
||||
mock_os_path.exists = lambda p: p.startswith("path_to_")
|
||||
mock_os_path.expanduser = lambda p: p + "_expanded"
|
||||
mock_os_path.realpath.side_effect = lambda p: "realpath_" + p
|
||||
results_iter = iter([self._make_result(["bar"]),
|
||||
self._make_result(["spam"])])
|
||||
mock_task_get.return_value.get_results.side_effect = results_iter
|
||||
mock_plot.trends.return_value = "rendered_trends_report"
|
||||
mock_fd = mock.mock_open(
|
||||
read_data="[\"result_1_from_file\", \"result_2_from_file\"]")
|
||||
mock_open.side_effect = mock_fd
|
||||
ret = self.task.trends(tasks=["ab123456-38d8-4c8f-bbcc-fc8f74b004ae",
|
||||
"cd654321-38d8-4c8f-bbcc-fc8f74b004ae",
|
||||
"path_to_file"],
|
||||
out="output.html", out_format="html")
|
||||
expected = [
|
||||
{"load_duration": 1.2, "full_duration": 2.3, "sla": "bar_sla",
|
||||
"key": {"name": "bar", "pos": 0}, "result": "bar_raw"},
|
||||
{"load_duration": 1.2, "full_duration": 2.3, "sla": "spam_sla",
|
||||
"key": {"name": "spam", "pos": 0}, "result": "spam_raw"},
|
||||
"result_1_from_file", "result_2_from_file"]
|
||||
mock_plot.trends.assert_called_once_with(expected)
|
||||
self.assertEqual([mock.call("path_to_file_expanded", "r"),
|
||||
mock.call("output.html_expanded", "w+")],
|
||||
mock_open.mock_calls)
|
||||
self.assertIsNone(ret)
|
||||
self.assertEqual([mock.call("result_1_from_file",
|
||||
task.api.Task.TASK_RESULT_SCHEMA),
|
||||
mock.call("result_2_from_file",
|
||||
task.api.Task.TASK_RESULT_SCHEMA)],
|
||||
mock_validate.mock_calls)
|
||||
self.assertEqual([mock.call("ab123456-38d8-4c8f-bbcc-fc8f74b004ae"),
|
||||
mock.call().get_results(),
|
||||
mock.call("cd654321-38d8-4c8f-bbcc-fc8f74b004ae"),
|
||||
mock.call().get_results()],
|
||||
mock_task_get.mock_calls)
|
||||
self.assertFalse(mock_webbrowser.open_new_tab.called)
|
||||
mock_fd.return_value.write.assert_called_once_with(
|
||||
"rendered_trends_report")
|
||||
|
||||
@mock.patch("rally.cli.commands.task.jsonschema.validate",
|
||||
return_value=None)
|
||||
@mock.patch("rally.cli.commands.task.os.path")
|
||||
@mock.patch("rally.cli.commands.task.open", create=True)
|
||||
@mock.patch("rally.cli.commands.task.plot")
|
||||
@mock.patch("rally.cli.commands.task.webbrowser")
|
||||
def test_trends_single_file_and_open_webbrowser(
|
||||
self, mock_webbrowser, mock_plot, mock_open, mock_os_path,
|
||||
mock_validate):
|
||||
mock_os_path.exists.return_value = True
|
||||
mock_os_path.expanduser = lambda path: path
|
||||
mock_os_path.realpath.side_effect = lambda p: "realpath_" + p
|
||||
mock_open.side_effect = mock.mock_open(read_data="[\"result\"]")
|
||||
ret = self.task.trends(tasks=["path_to_file"], open_it=True,
|
||||
out="output.html", out_format="html")
|
||||
self.assertIsNone(ret)
|
||||
mock_webbrowser.open_new_tab.assert_called_once_with(
|
||||
"file://realpath_output.html")
|
||||
|
||||
@mock.patch("rally.cli.commands.task.os.path")
|
||||
@mock.patch("rally.cli.commands.task.open", create=True)
|
||||
@mock.patch("rally.cli.commands.task.plot")
|
||||
@mock.patch("rally.cli.commands.task.api.Task.get")
|
||||
def test_trends_task_id_is_not_uuid_like(self, mock_task_get, mock_plot,
|
||||
mock_open, mock_os_path):
|
||||
mock_os_path.exists.return_value = False
|
||||
mock_task_get.return_value.get_results.return_value = (
|
||||
self._make_result(["foo"]))
|
||||
|
||||
ret = self.task.trends(tasks=["ab123456-38d8-4c8f-bbcc-fc8f74b004ae"],
|
||||
out="output.html", out_format="html")
|
||||
self.assertIsNone(ret)
|
||||
|
||||
ret = self.task.trends(tasks=["this-is-not-uuid"],
|
||||
out="output.html", out_format="html")
|
||||
self.assertEqual(1, ret)
|
||||
|
||||
@mock.patch("rally.cli.commands.task.os.path")
|
||||
@mock.patch("rally.cli.commands.task.open", create=True)
|
||||
@mock.patch("rally.cli.commands.task.plot")
|
||||
def test_trends_wrong_results_format(self, mock_plot,
|
||||
mock_open, mock_os_path):
|
||||
mock_os_path.exists.return_value = True
|
||||
mock_open.side_effect = mock.mock_open(read_data="[42]")
|
||||
ret = self.task.trends(tasks=["path_to_file"],
|
||||
out="output.html", out_format="html")
|
||||
self.assertEqual(1, ret)
|
||||
|
||||
with mock.patch("rally.cli.commands.task.api.Task.TASK_RESULT_SCHEMA",
|
||||
{"type": "number"}):
|
||||
ret = self.task.trends(tasks=["path_to_file"],
|
||||
out="output.html", out_format="html")
|
||||
self.assertIsNone(ret)
|
||||
|
||||
def test_trends_no_tasks_given(self):
|
||||
ret = self.task.trends(tasks=[],
|
||||
out="output.html", out_format="html")
|
||||
self.assertEqual(1, ret)
|
||||
|
||||
@mock.patch("rally.cli.commands.task.jsonschema.validate",
|
||||
return_value=None)
|
||||
@mock.patch("rally.cli.commands.task.os.path.realpath",
|
||||
|
||||
@ -150,6 +150,25 @@ class PlotTestCase(test.TestCase):
|
||||
def test__extend_results_empty(self):
|
||||
self.assertEqual([], plot._extend_results([]))
|
||||
|
||||
@mock.patch(PLOT + "Trends")
|
||||
@mock.patch(PLOT + "ui_utils.get_template")
|
||||
@mock.patch(PLOT + "_extend_results")
|
||||
def test_trends(self, mock__extend_results, mock_get_template,
|
||||
mock_trends):
|
||||
mock__extend_results.return_value = ["foo", "bar"]
|
||||
trends = mock.Mock()
|
||||
trends.get_data.return_value = ["foo", "bar"]
|
||||
mock_trends.return_value = trends
|
||||
template = mock.Mock()
|
||||
template.render.return_value = "trends html"
|
||||
mock_get_template.return_value = template
|
||||
|
||||
self.assertEqual("trends html", plot.trends("tasks_results"))
|
||||
self.assertEqual([mock.call("foo"), mock.call("bar")],
|
||||
trends.add_result.mock_calls)
|
||||
mock_get_template.assert_called_once_with("task/trends.html")
|
||||
template.render.assert_called_once_with(data="[\"foo\", \"bar\"]")
|
||||
|
||||
|
||||
@ddt.ddt
|
||||
class TrendsTestCase(test.TestCase):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user