[Reports][CLI] Introduce Trends report

New HTML report for statistics trends for given tasks
which is generated by cli command "rally task trends".

Blueprint: trends-report
Change-Id: I0805058e8bd225796b02516fad094d73037d5495

etc/rally.bash_completion

@@ -43,6 +43,7 @@ _rally()
     OPTS["task_sla_check"]="--uuid --json"
     OPTS["task_start"]="--deployment --task --task-args --task-args-file --tag --no-use --abort-on-sla-failure"
     OPTS["task_status"]="--uuid"
+    OPTS["task_trends"]="--out --open --tasks"
     OPTS["task_use"]="--uuid"
     OPTS["task_validate"]="--deployment --task --task-args --task-args-file"
     OPTS["verify_compare"]="--uuid-1 --uuid-2 --csv --html --json --output-file --threshold"
@@ -87,4 +88,4 @@ _rally()
     return 0
 }
 
-complete -o filenames -F _rally rally
+complete -o filenames -F _rally rally

rally/cli/commands/task.py

@@ -543,6 +543,67 @@ class TaskCommands(object):
                 print(_("There are no tasks. To run a new task, use:\n"
                         "\trally task start"))
 
+    @cliutils.args("--out", metavar="<path>",
+                   type=str, dest="out", required=False,
+                   help="Path to output file.")
+    @cliutils.args("--open", dest="open_it", action="store_true",
+                   help="Open the output in a browser.")
+    @cliutils.args("--tasks", dest="tasks", nargs="+",
+                   help="UUIDs of tasks, or JSON files with task results")
+    @cliutils.suppress_warnings
+    def trends(self, *args, **kwargs):
+        """Generate workloads trends HTML report."""
+        tasks = kwargs.get("tasks", []) or list(args)
+
+        if not tasks:
+            print(_("ERROR: At least one task must be specified"),
+                  file=sys.stderr)
+            return 1
+
+        results = []
+        for task_id in tasks:
+            if os.path.exists(os.path.expanduser(task_id)):
+                with open(os.path.expanduser(task_id), "r") as inp_js:
+                    task_results = json.load(inp_js)
+                    for result in task_results:
+                        try:
+                            jsonschema.validate(
+                                result,
+                                api.Task.TASK_RESULT_SCHEMA)
+                        except jsonschema.ValidationError as e:
+                            print(_("ERROR: Invalid task result format in %s")
+                                  % task_id, file=sys.stderr)
+                            print(six.text_type(e), file=sys.stderr)
+                            return 1
+
+            elif uuidutils.is_uuid_like(task_id):
+                task_results = map(
+                    lambda x: {"key": x["key"],
+                               "sla": x["data"]["sla"],
+                               "result": x["data"]["raw"],
+                               "load_duration": x["data"]["load_duration"],
+                               "full_duration": x["data"]["full_duration"]},
+                    api.Task.get(task_id).get_results())
+            else:
+                print(_("ERROR: Invalid UUID or file name passed: %s")
+                      % task_id, file=sys.stderr)
+                return 1
+
+            results.extend(task_results)
+
+        result = plot.trends(results)
+
+        out = kwargs.get("out")
+        if out:
+            output_file = os.path.expanduser(out)
+
+            with open(output_file, "w+") as f:
+                f.write(result)
+            if kwargs.get("open_it"):
+                webbrowser.open_new_tab("file://" + os.path.realpath(out))
+        else:
+            print(result)
+
     @cliutils.args("--tasks", dest="tasks", nargs="+",
                    help="UUIDs of tasks, or JSON files with task results")
     @cliutils.args("--out", metavar="<path>",

rally/task/processing/plot.py

@@ -154,6 +154,14 @@ def plot(tasks_results, include_libs=False):
                            include_libs=include_libs)
 
 
+def trends(tasks_results):
+    trends = Trends()
+    for i, scenario in enumerate(_extend_results(tasks_results), 1):
+        trends.add_result(scenario)
+    template = ui_utils.get_template("task/trends.html")
+    return template.render(data=json.dumps(trends.get_data()))
+
+
 class Trends(object):
     """Process tasks results and make trends data.
 
@@ -242,7 +250,8 @@ class Trends(object):
                                ("max", charts.streaming.MaxComputation()),
                                ("avg", charts.streaming.MeanComputation())):
                 for k, v in total[stat]:
-                    comp.add(v)
+                    if isinstance(v, (float,) + six.integer_types):
+                        comp.add(v)
                 self._tasks[key]["stat"][stat] = comp.result()
             del self._tasks[key]["data"]
             self._tasks[key]["total"] = list(total.items())

tests/functional/test_cli_task.py

@@ -317,6 +317,40 @@ class TaskTestCase(unittest.TestCase):
         self.assertTrue(os.path.exists(html_report))
         self._assert_html_report_libs_are_embedded(html_report)
 
+    def test_trends(self):
+        cfg1 = {
+            "Dummy.dummy": [
+                {"runner": {"type": "constant", "times": 2,
+                            "concurrency": 2}}],
+            "Dummy.dummy_random_action": [
+                {"args": {"actions_num": 4},
+                 "runner": {"type": "constant", "times": 2, "concurrency": 2}},
+                {"runner": {"type": "constant", "times": 2,
+                            "concurrency": 2}}]}
+        cfg2 = {
+            "Dummy.dummy": [
+                {"args": {"sleep": 0.6},
+                 "runner": {"type": "constant", "times": 2,
+                            "concurrency": 2}}]}
+
+        config1 = utils.TaskConfig(cfg1)
+        config2 = utils.TaskConfig(cfg2)
+        rally = utils.Rally()
+        report = rally.gen_report_path(extension="html")
+
+        for i in range(5):
+            rally("task start --task %(file)s --tag trends_run_%(idx)d"
+                  % {"file": config1.filename, "idx": i})
+        rally("task start --task %s --tag trends_run_once" % config2.filename)
+
+        tasks_list = rally("task list")
+        uuids = [u[2:38] for u in tasks_list.split("\n") if "trends_run" in u]
+
+        rally("task trends %(uuids)s --out %(report)s"
+              % {"uuids": " ".join(uuids), "report": report})
+        del config1, config2
+        self.assertTrue(os.path.exists(report))
+
     def test_delete(self):
         rally = utils.Rally()
         cfg = self._get_sample_task_config()

tests/unit/cli/commands/test_task.py

@@ -422,6 +422,120 @@ class TaskCommandsTestCase(test.TestCase):
                        consts.TaskStatus.ABORTED)))
         mock_stdout.write.assert_has_calls([mock.call(expected_out)])
 
+    def _make_result(self, keys):
+        return [{"key": {"name": key, "pos": 0},
+                 "data": {"raw": key + "_raw",
+                          "sla": key + "_sla",
+                          "load_duration": 1.2,
+                          "full_duration": 2.3}} for key in keys]
+
+    @mock.patch("rally.cli.commands.task.jsonschema.validate",
+                return_value=None)
+    @mock.patch("rally.cli.commands.task.os.path")
+    @mock.patch("rally.cli.commands.task.open", create=True)
+    @mock.patch("rally.cli.commands.task.plot")
+    @mock.patch("rally.cli.commands.task.api.Task.get")
+    @mock.patch("rally.cli.commands.task.webbrowser")
+    def test_trends(self, mock_webbrowser, mock_task_get, mock_plot,
+                    mock_open, mock_os_path, mock_validate):
+        mock_os_path.exists = lambda p: p.startswith("path_to_")
+        mock_os_path.expanduser = lambda p: p + "_expanded"
+        mock_os_path.realpath.side_effect = lambda p: "realpath_" + p
+        results_iter = iter([self._make_result(["bar"]),
+                             self._make_result(["spam"])])
+        mock_task_get.return_value.get_results.side_effect = results_iter
+        mock_plot.trends.return_value = "rendered_trends_report"
+        mock_fd = mock.mock_open(
+            read_data="[\"result_1_from_file\", \"result_2_from_file\"]")
+        mock_open.side_effect = mock_fd
+        ret = self.task.trends(tasks=["ab123456-38d8-4c8f-bbcc-fc8f74b004ae",
+                                      "cd654321-38d8-4c8f-bbcc-fc8f74b004ae",
+                                      "path_to_file"],
+                               out="output.html", out_format="html")
+        expected = [
+            {"load_duration": 1.2, "full_duration": 2.3, "sla": "bar_sla",
+             "key": {"name": "bar", "pos": 0}, "result": "bar_raw"},
+            {"load_duration": 1.2, "full_duration": 2.3, "sla": "spam_sla",
+             "key": {"name": "spam", "pos": 0}, "result": "spam_raw"},
+            "result_1_from_file", "result_2_from_file"]
+        mock_plot.trends.assert_called_once_with(expected)
+        self.assertEqual([mock.call("path_to_file_expanded", "r"),
+                          mock.call("output.html_expanded", "w+")],
+                         mock_open.mock_calls)
+        self.assertIsNone(ret)
+        self.assertEqual([mock.call("result_1_from_file",
+                                    task.api.Task.TASK_RESULT_SCHEMA),
+                          mock.call("result_2_from_file",
+                                    task.api.Task.TASK_RESULT_SCHEMA)],
+                         mock_validate.mock_calls)
+        self.assertEqual([mock.call("ab123456-38d8-4c8f-bbcc-fc8f74b004ae"),
+                          mock.call().get_results(),
+                          mock.call("cd654321-38d8-4c8f-bbcc-fc8f74b004ae"),
+                          mock.call().get_results()],
+                         mock_task_get.mock_calls)
+        self.assertFalse(mock_webbrowser.open_new_tab.called)
+        mock_fd.return_value.write.assert_called_once_with(
+            "rendered_trends_report")
+
+    @mock.patch("rally.cli.commands.task.jsonschema.validate",
+                return_value=None)
+    @mock.patch("rally.cli.commands.task.os.path")
+    @mock.patch("rally.cli.commands.task.open", create=True)
+    @mock.patch("rally.cli.commands.task.plot")
+    @mock.patch("rally.cli.commands.task.webbrowser")
+    def test_trends_single_file_and_open_webbrowser(
+            self, mock_webbrowser, mock_plot, mock_open, mock_os_path,
+            mock_validate):
+        mock_os_path.exists.return_value = True
+        mock_os_path.expanduser = lambda path: path
+        mock_os_path.realpath.side_effect = lambda p: "realpath_" + p
+        mock_open.side_effect = mock.mock_open(read_data="[\"result\"]")
+        ret = self.task.trends(tasks=["path_to_file"], open_it=True,
+                               out="output.html", out_format="html")
+        self.assertIsNone(ret)
+        mock_webbrowser.open_new_tab.assert_called_once_with(
+            "file://realpath_output.html")
+
+    @mock.patch("rally.cli.commands.task.os.path")
+    @mock.patch("rally.cli.commands.task.open", create=True)
+    @mock.patch("rally.cli.commands.task.plot")
+    @mock.patch("rally.cli.commands.task.api.Task.get")
+    def test_trends_task_id_is_not_uuid_like(self, mock_task_get, mock_plot,
+                                             mock_open, mock_os_path):
+        mock_os_path.exists.return_value = False
+        mock_task_get.return_value.get_results.return_value = (
+            self._make_result(["foo"]))
+
+        ret = self.task.trends(tasks=["ab123456-38d8-4c8f-bbcc-fc8f74b004ae"],
+                               out="output.html", out_format="html")
+        self.assertIsNone(ret)
+
+        ret = self.task.trends(tasks=["this-is-not-uuid"],
+                               out="output.html", out_format="html")
+        self.assertEqual(1, ret)
+
+    @mock.patch("rally.cli.commands.task.os.path")
+    @mock.patch("rally.cli.commands.task.open", create=True)
+    @mock.patch("rally.cli.commands.task.plot")
+    def test_trends_wrong_results_format(self, mock_plot,
+                                         mock_open, mock_os_path):
+        mock_os_path.exists.return_value = True
+        mock_open.side_effect = mock.mock_open(read_data="[42]")
+        ret = self.task.trends(tasks=["path_to_file"],
+                               out="output.html", out_format="html")
+        self.assertEqual(1, ret)
+
+        with mock.patch("rally.cli.commands.task.api.Task.TASK_RESULT_SCHEMA",
+                        {"type": "number"}):
+            ret = self.task.trends(tasks=["path_to_file"],
+                                   out="output.html", out_format="html")
+            self.assertIsNone(ret)
+
+    def test_trends_no_tasks_given(self):
+        ret = self.task.trends(tasks=[],
+                               out="output.html", out_format="html")
+        self.assertEqual(1, ret)
+
     @mock.patch("rally.cli.commands.task.jsonschema.validate",
                 return_value=None)
     @mock.patch("rally.cli.commands.task.os.path.realpath",

tests/unit/task/processing/test_plot.py

@@ -150,6 +150,25 @@ class PlotTestCase(test.TestCase):
     def test__extend_results_empty(self):
         self.assertEqual([], plot._extend_results([]))
 
+    @mock.patch(PLOT + "Trends")
+    @mock.patch(PLOT + "ui_utils.get_template")
+    @mock.patch(PLOT + "_extend_results")
+    def test_trends(self, mock__extend_results, mock_get_template,
+                    mock_trends):
+        mock__extend_results.return_value = ["foo", "bar"]
+        trends = mock.Mock()
+        trends.get_data.return_value = ["foo", "bar"]
+        mock_trends.return_value = trends
+        template = mock.Mock()
+        template.render.return_value = "trends html"
+        mock_get_template.return_value = template
+
+        self.assertEqual("trends html", plot.trends("tasks_results"))
+        self.assertEqual([mock.call("foo"), mock.call("bar")],
+                         trends.add_result.mock_calls)
+        mock_get_template.assert_called_once_with("task/trends.html")
+        template.render.assert_called_once_with(data="[\"foo\", \"bar\"]")
+
 
 @ddt.ddt
 class TrendsTestCase(test.TestCase):