From b7fa1422f6de7a0543f54cbbd379dafda351f263 Mon Sep 17 00:00:00 2001 From: Alexander Maretskiy Date: Thu, 17 Mar 2016 18:09:13 +0200 Subject: [PATCH] [Reports][CLI] Introduce Trends report New HTML report for statistics trends for given tasks which is generated by cli command "rally task trends". Blueprint: trends-report Change-Id: I0805058e8bd225796b02516fad094d73037d5495 --- etc/rally.bash_completion | 3 +- rally/cli/commands/task.py | 61 +++++++++++++ rally/task/processing/plot.py | 11 ++- tests/functional/test_cli_task.py | 34 +++++++ tests/unit/cli/commands/test_task.py | 114 ++++++++++++++++++++++++ tests/unit/task/processing/test_plot.py | 19 ++++ 6 files changed, 240 insertions(+), 2 deletions(-) diff --git a/etc/rally.bash_completion b/etc/rally.bash_completion index b84eefb0ac..9d588e0f4f 100644 --- a/etc/rally.bash_completion +++ b/etc/rally.bash_completion @@ -43,6 +43,7 @@ _rally() OPTS["task_sla_check"]="--uuid --json" OPTS["task_start"]="--deployment --task --task-args --task-args-file --tag --no-use --abort-on-sla-failure" OPTS["task_status"]="--uuid" + OPTS["task_trends"]="--out --open --tasks" OPTS["task_use"]="--uuid" OPTS["task_validate"]="--deployment --task --task-args --task-args-file" OPTS["verify_compare"]="--uuid-1 --uuid-2 --csv --html --json --output-file --threshold" @@ -87,4 +88,4 @@ _rally() return 0 } -complete -o filenames -F _rally rally \ No newline at end of file +complete -o filenames -F _rally rally diff --git a/rally/cli/commands/task.py b/rally/cli/commands/task.py index fba70c4464..88d712888a 100644 --- a/rally/cli/commands/task.py +++ b/rally/cli/commands/task.py @@ -543,6 +543,67 @@ class TaskCommands(object): print(_("There are no tasks. To run a new task, use:\n" "\trally task start")) + @cliutils.args("--out", metavar="", + type=str, dest="out", required=False, + help="Path to output file.") + @cliutils.args("--open", dest="open_it", action="store_true", + help="Open the output in a browser.") + @cliutils.args("--tasks", dest="tasks", nargs="+", + help="UUIDs of tasks, or JSON files with task results") + @cliutils.suppress_warnings + def trends(self, *args, **kwargs): + """Generate workloads trends HTML report.""" + tasks = kwargs.get("tasks", []) or list(args) + + if not tasks: + print(_("ERROR: At least one task must be specified"), + file=sys.stderr) + return 1 + + results = [] + for task_id in tasks: + if os.path.exists(os.path.expanduser(task_id)): + with open(os.path.expanduser(task_id), "r") as inp_js: + task_results = json.load(inp_js) + for result in task_results: + try: + jsonschema.validate( + result, + api.Task.TASK_RESULT_SCHEMA) + except jsonschema.ValidationError as e: + print(_("ERROR: Invalid task result format in %s") + % task_id, file=sys.stderr) + print(six.text_type(e), file=sys.stderr) + return 1 + + elif uuidutils.is_uuid_like(task_id): + task_results = map( + lambda x: {"key": x["key"], + "sla": x["data"]["sla"], + "result": x["data"]["raw"], + "load_duration": x["data"]["load_duration"], + "full_duration": x["data"]["full_duration"]}, + api.Task.get(task_id).get_results()) + else: + print(_("ERROR: Invalid UUID or file name passed: %s") + % task_id, file=sys.stderr) + return 1 + + results.extend(task_results) + + result = plot.trends(results) + + out = kwargs.get("out") + if out: + output_file = os.path.expanduser(out) + + with open(output_file, "w+") as f: + f.write(result) + if kwargs.get("open_it"): + webbrowser.open_new_tab("file://" + os.path.realpath(out)) + else: + print(result) + @cliutils.args("--tasks", dest="tasks", nargs="+", help="UUIDs of tasks, or JSON files with task results") @cliutils.args("--out", metavar="", diff --git a/rally/task/processing/plot.py b/rally/task/processing/plot.py index 4f368fee12..e2040d966d 100644 --- a/rally/task/processing/plot.py +++ b/rally/task/processing/plot.py @@ -154,6 +154,14 @@ def plot(tasks_results, include_libs=False): include_libs=include_libs) +def trends(tasks_results): + trends = Trends() + for i, scenario in enumerate(_extend_results(tasks_results), 1): + trends.add_result(scenario) + template = ui_utils.get_template("task/trends.html") + return template.render(data=json.dumps(trends.get_data())) + + class Trends(object): """Process tasks results and make trends data. @@ -242,7 +250,8 @@ class Trends(object): ("max", charts.streaming.MaxComputation()), ("avg", charts.streaming.MeanComputation())): for k, v in total[stat]: - comp.add(v) + if isinstance(v, (float,) + six.integer_types): + comp.add(v) self._tasks[key]["stat"][stat] = comp.result() del self._tasks[key]["data"] self._tasks[key]["total"] = list(total.items()) diff --git a/tests/functional/test_cli_task.py b/tests/functional/test_cli_task.py index f171f172d5..bdd6cea8db 100644 --- a/tests/functional/test_cli_task.py +++ b/tests/functional/test_cli_task.py @@ -317,6 +317,40 @@ class TaskTestCase(unittest.TestCase): self.assertTrue(os.path.exists(html_report)) self._assert_html_report_libs_are_embedded(html_report) + def test_trends(self): + cfg1 = { + "Dummy.dummy": [ + {"runner": {"type": "constant", "times": 2, + "concurrency": 2}}], + "Dummy.dummy_random_action": [ + {"args": {"actions_num": 4}, + "runner": {"type": "constant", "times": 2, "concurrency": 2}}, + {"runner": {"type": "constant", "times": 2, + "concurrency": 2}}]} + cfg2 = { + "Dummy.dummy": [ + {"args": {"sleep": 0.6}, + "runner": {"type": "constant", "times": 2, + "concurrency": 2}}]} + + config1 = utils.TaskConfig(cfg1) + config2 = utils.TaskConfig(cfg2) + rally = utils.Rally() + report = rally.gen_report_path(extension="html") + + for i in range(5): + rally("task start --task %(file)s --tag trends_run_%(idx)d" + % {"file": config1.filename, "idx": i}) + rally("task start --task %s --tag trends_run_once" % config2.filename) + + tasks_list = rally("task list") + uuids = [u[2:38] for u in tasks_list.split("\n") if "trends_run" in u] + + rally("task trends %(uuids)s --out %(report)s" + % {"uuids": " ".join(uuids), "report": report}) + del config1, config2 + self.assertTrue(os.path.exists(report)) + def test_delete(self): rally = utils.Rally() cfg = self._get_sample_task_config() diff --git a/tests/unit/cli/commands/test_task.py b/tests/unit/cli/commands/test_task.py index fcfa6400d4..2e95fffc4e 100644 --- a/tests/unit/cli/commands/test_task.py +++ b/tests/unit/cli/commands/test_task.py @@ -422,6 +422,120 @@ class TaskCommandsTestCase(test.TestCase): consts.TaskStatus.ABORTED))) mock_stdout.write.assert_has_calls([mock.call(expected_out)]) + def _make_result(self, keys): + return [{"key": {"name": key, "pos": 0}, + "data": {"raw": key + "_raw", + "sla": key + "_sla", + "load_duration": 1.2, + "full_duration": 2.3}} for key in keys] + + @mock.patch("rally.cli.commands.task.jsonschema.validate", + return_value=None) + @mock.patch("rally.cli.commands.task.os.path") + @mock.patch("rally.cli.commands.task.open", create=True) + @mock.patch("rally.cli.commands.task.plot") + @mock.patch("rally.cli.commands.task.api.Task.get") + @mock.patch("rally.cli.commands.task.webbrowser") + def test_trends(self, mock_webbrowser, mock_task_get, mock_plot, + mock_open, mock_os_path, mock_validate): + mock_os_path.exists = lambda p: p.startswith("path_to_") + mock_os_path.expanduser = lambda p: p + "_expanded" + mock_os_path.realpath.side_effect = lambda p: "realpath_" + p + results_iter = iter([self._make_result(["bar"]), + self._make_result(["spam"])]) + mock_task_get.return_value.get_results.side_effect = results_iter + mock_plot.trends.return_value = "rendered_trends_report" + mock_fd = mock.mock_open( + read_data="[\"result_1_from_file\", \"result_2_from_file\"]") + mock_open.side_effect = mock_fd + ret = self.task.trends(tasks=["ab123456-38d8-4c8f-bbcc-fc8f74b004ae", + "cd654321-38d8-4c8f-bbcc-fc8f74b004ae", + "path_to_file"], + out="output.html", out_format="html") + expected = [ + {"load_duration": 1.2, "full_duration": 2.3, "sla": "bar_sla", + "key": {"name": "bar", "pos": 0}, "result": "bar_raw"}, + {"load_duration": 1.2, "full_duration": 2.3, "sla": "spam_sla", + "key": {"name": "spam", "pos": 0}, "result": "spam_raw"}, + "result_1_from_file", "result_2_from_file"] + mock_plot.trends.assert_called_once_with(expected) + self.assertEqual([mock.call("path_to_file_expanded", "r"), + mock.call("output.html_expanded", "w+")], + mock_open.mock_calls) + self.assertIsNone(ret) + self.assertEqual([mock.call("result_1_from_file", + task.api.Task.TASK_RESULT_SCHEMA), + mock.call("result_2_from_file", + task.api.Task.TASK_RESULT_SCHEMA)], + mock_validate.mock_calls) + self.assertEqual([mock.call("ab123456-38d8-4c8f-bbcc-fc8f74b004ae"), + mock.call().get_results(), + mock.call("cd654321-38d8-4c8f-bbcc-fc8f74b004ae"), + mock.call().get_results()], + mock_task_get.mock_calls) + self.assertFalse(mock_webbrowser.open_new_tab.called) + mock_fd.return_value.write.assert_called_once_with( + "rendered_trends_report") + + @mock.patch("rally.cli.commands.task.jsonschema.validate", + return_value=None) + @mock.patch("rally.cli.commands.task.os.path") + @mock.patch("rally.cli.commands.task.open", create=True) + @mock.patch("rally.cli.commands.task.plot") + @mock.patch("rally.cli.commands.task.webbrowser") + def test_trends_single_file_and_open_webbrowser( + self, mock_webbrowser, mock_plot, mock_open, mock_os_path, + mock_validate): + mock_os_path.exists.return_value = True + mock_os_path.expanduser = lambda path: path + mock_os_path.realpath.side_effect = lambda p: "realpath_" + p + mock_open.side_effect = mock.mock_open(read_data="[\"result\"]") + ret = self.task.trends(tasks=["path_to_file"], open_it=True, + out="output.html", out_format="html") + self.assertIsNone(ret) + mock_webbrowser.open_new_tab.assert_called_once_with( + "file://realpath_output.html") + + @mock.patch("rally.cli.commands.task.os.path") + @mock.patch("rally.cli.commands.task.open", create=True) + @mock.patch("rally.cli.commands.task.plot") + @mock.patch("rally.cli.commands.task.api.Task.get") + def test_trends_task_id_is_not_uuid_like(self, mock_task_get, mock_plot, + mock_open, mock_os_path): + mock_os_path.exists.return_value = False + mock_task_get.return_value.get_results.return_value = ( + self._make_result(["foo"])) + + ret = self.task.trends(tasks=["ab123456-38d8-4c8f-bbcc-fc8f74b004ae"], + out="output.html", out_format="html") + self.assertIsNone(ret) + + ret = self.task.trends(tasks=["this-is-not-uuid"], + out="output.html", out_format="html") + self.assertEqual(1, ret) + + @mock.patch("rally.cli.commands.task.os.path") + @mock.patch("rally.cli.commands.task.open", create=True) + @mock.patch("rally.cli.commands.task.plot") + def test_trends_wrong_results_format(self, mock_plot, + mock_open, mock_os_path): + mock_os_path.exists.return_value = True + mock_open.side_effect = mock.mock_open(read_data="[42]") + ret = self.task.trends(tasks=["path_to_file"], + out="output.html", out_format="html") + self.assertEqual(1, ret) + + with mock.patch("rally.cli.commands.task.api.Task.TASK_RESULT_SCHEMA", + {"type": "number"}): + ret = self.task.trends(tasks=["path_to_file"], + out="output.html", out_format="html") + self.assertIsNone(ret) + + def test_trends_no_tasks_given(self): + ret = self.task.trends(tasks=[], + out="output.html", out_format="html") + self.assertEqual(1, ret) + @mock.patch("rally.cli.commands.task.jsonschema.validate", return_value=None) @mock.patch("rally.cli.commands.task.os.path.realpath", diff --git a/tests/unit/task/processing/test_plot.py b/tests/unit/task/processing/test_plot.py index ba0c9b0873..52bd1a71c9 100644 --- a/tests/unit/task/processing/test_plot.py +++ b/tests/unit/task/processing/test_plot.py @@ -150,6 +150,25 @@ class PlotTestCase(test.TestCase): def test__extend_results_empty(self): self.assertEqual([], plot._extend_results([])) + @mock.patch(PLOT + "Trends") + @mock.patch(PLOT + "ui_utils.get_template") + @mock.patch(PLOT + "_extend_results") + def test_trends(self, mock__extend_results, mock_get_template, + mock_trends): + mock__extend_results.return_value = ["foo", "bar"] + trends = mock.Mock() + trends.get_data.return_value = ["foo", "bar"] + mock_trends.return_value = trends + template = mock.Mock() + template.render.return_value = "trends html" + mock_get_template.return_value = template + + self.assertEqual("trends html", plot.trends("tasks_results")) + self.assertEqual([mock.call("foo"), mock.call("bar")], + trends.add_result.mock_calls) + mock_get_template.assert_called_once_with("task/trends.html") + template.render.assert_called_once_with(data="[\"foo\", \"bar\"]") + @ddt.ddt class TrendsTestCase(test.TestCase):