[Hooks][Reports] Show Hooks output in HTML report

This adds support of hooks output (both "additive"
and "complete") to HTML report.

Tab "Hooks" with output charts appears in case if
hook has saved some output.

Change-Id: I9f14cec23da51ff1603d04338b8f95e6ae2a6801

rally-jobs/extra/hook_example_script.sh

@@ -0,0 +1,56 @@
+#!/bin/sh
+
+rand_int() {
+    od -An -tu -N1 /dev/urandom | tr -d ' '
+}
+
+cat << EOF
+{
+  "additive": [
+    {
+      "title": "Statistics table from Hook",
+      "chart_plugin": "StatsTable",
+      "data": [
+        ["Alice", $(rand_int)],
+        ["Bob", $(rand_int)],
+        ["Carol", $(rand_int)]]
+    },
+    {
+      "title": "StackedArea chart from Hook",
+      "description": "This is generated by ${0}",
+      "chart_plugin": "StackedArea",
+      "data": [
+        ["Alpha", $(rand_int)],
+        ["Beta", $(rand_int)],
+        ["Gamma", $(rand_int)]]
+    }
+  ],
+  "complete": [
+    {
+      "title": "Lines chart from Hook",
+      "description": "Random data generated by ${0}",
+      "chart_plugin": "Lines",
+      "axis_label": "X-axis label",
+      "label": "Y-axis label",
+      "data": [
+        ["Foo", [[1, $(rand_int)], [2, $(rand_int)], [3, $(rand_int)], [4, $(rand_int)], [5, $(rand_int)]]],
+        ["Bar", [[1, $(rand_int)], [2, $(rand_int)], [3, $(rand_int)], [4, $(rand_int)], [5, $(rand_int)]]],
+        ["Spam", [[1, $(rand_int)], [2, $(rand_int)], [3, $(rand_int)], [4, $(rand_int)], [5, $(rand_int)]]],
+        ["Quiz", [[1, $(rand_int)], [2, $(rand_int)], [3, $(rand_int)], [4, $(rand_int)], [5, $(rand_int)]]]
+      ]
+    },
+    {
+      "title": "Pie chart from Hook",
+      "description": "Yet another data generated by ${0}",
+      "chart_plugin": "Pie",
+      "data": [
+        ["Cat", $(rand_int)],
+        ["Tiger", $(rand_int)],
+        ["Jaguar", $(rand_int)],
+        ["Panther", $(rand_int)],
+        ["Lynx", $(rand_int)]
+      ]
+    }
+  ]
+}
+EOF

rally-jobs/rally.yaml

@@ -561,20 +561,36 @@
 
     -
       args:
-        sleep: 0.25
+        sleep: 0.75
       runner:
         type: "constant"
-        times: 10
+        times: 20
         concurrency: 2
       hooks:
         - name: sys_call
-          description: test hook
+          description: Run script
-          args: /bin/true
+          args: sh /home/jenkins/.rally/extra/hook_example_script.sh
           trigger:
             name: event
             args:
               unit: iteration
-              at: [2, 4, 6, 8, 10]
+              at: [2, 5, 8, 13, 17]
+        - name: sys_call
+          description: Show time
+          args: date +%Y-%m-%dT%H:%M:%S
+          trigger:
+            name: event
+            args:
+              unit: time
+              at: [0, 2, 5, 6, 9]
+        - name: sys_call
+          description: Show system name
+          args: uname -a
+          trigger:
+            name: event
+            args:
+              unit: iteration
+              at: [2, 3, 4, 5, 6, 8, 10, 12, 13, 15, 17, 18]
       sla:
         failure_rate:
           max: 0
@@ -610,8 +626,8 @@
         concurrency: 1
       hooks:
         - name: sys_call
-          description: test hook
+          description: Get system name
-          args: /bin/true
+          args: uname -a
           trigger:
             name: event
             args:

rally/common/objects/task.py

@@ -82,7 +82,8 @@ OUTPUT_SCHEMA = {
                              }
                          },
                          "required": ["cols", "rows"],
-                         "additionalProperties": False}
+                         "additionalProperties": False},
+                        {"type": "array", "items": {"type": "string"}},
                     ]},
                     "label": {"type": "string"},
                     "axis_label": {"type": "string"}

rally/plugins/common/hook/sys_call.py

@@ -13,11 +13,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import json
 import shlex
 import subprocess
 
 from rally.common import logging
 from rally import consts
+from rally import exceptions
 from rally.task import hook
 
 
@@ -37,15 +39,28 @@ class SysCallHook(hook.Hook):
         LOG.debug("sys_call hook: Running command %s", self.config)
         proc = subprocess.Popen(shlex.split(self.config),
                                 stdout=subprocess.PIPE,
-                                stderr=subprocess.STDOUT)
+                                stderr=subprocess.PIPE)
-        proc.wait()
+        out, err = proc.communicate()
         LOG.debug("sys_call hook: Command %s returned %s",
                   self.config, proc.returncode)
-        if proc.returncode == 0:
+        if proc.returncode:
-            self.set_output(proc.stdout.read().decode())
-        else:
             self.set_error(
                 exception_name="n/a",  # no exception class
                 description="Subprocess returned {}".format(proc.returncode),
-                details=proc.stdout.read().decode(),
+                details=(err or "stdout: %s" % out))
-            )
+
+        # NOTE(amaretskiy): Try to load JSON for charts,
+        #                   otherwise save output as-is
+        try:
+            output = json.loads(out)
+            for arg in ("additive", "complete"):
+                for out_ in output.get(arg, []):
+                    self.add_output(**{arg: out_})
+        except (TypeError, ValueError, exceptions.RallyException):
+            self.add_output(
+                complete={"title": "System call",
+                          "chart_plugin": "TextArea",
+                          "description": "Args: %s" % self.config,
+                          "data": ["RetCode: %i" % proc.returncode,
+                                   "StdOut: %s" % (out or "(empty)"),
+                                   "StdErr: %s" % (err or "(empty)")]})

rally/task/hook.py

@@ -25,6 +25,8 @@ from rally.common import logging
 from rally.common.plugin import plugin
 from rally.common import utils as rutils
 from rally import consts
+from rally import exceptions
+from rally.task.processing import charts
 from rally.task import trigger
 from rally.task import utils
 
@@ -152,13 +154,21 @@ class Hook(plugin.Plugin):
         """Set status to result."""
         self._result["status"] = status
 
-    def set_output(self, output):
+    def add_output(self, additive=None, complete=None):
-        """Set output to result.
+        """Save custom output.
 
-        :param output: Diagram data in task.OUTPUT_SCHEMA format
+        :param additive: dict with additive output
+        :param complete: dict with complete output
+        :raises RallyException: if output has wrong format
         """
-        if output:
+        if "output" not in self._result:
-            self._result["output"] = output
+            self._result["output"] = {"additive": [], "complete": []}
+        for key, value in (("additive", additive), ("complete", complete)):
+            if value:
+                message = charts.validate_output(key, value)
+                if message:
+                    raise exceptions.RallyException(message)
+                self._result["output"][key].append(value)
 
     def run_async(self):
         """Run hook asynchronously."""
@@ -190,7 +200,7 @@ class Hook(plugin.Plugin):
         Optionally the following methods should be called:
             set_error - to indicate that there was an error;
                         automatically sets hook execution status to 'failed'
-            set_output - to provide diagram data
+            add_output - provide data for report
         """
 
     def result(self):

rally/task/processing/plot.py

@@ -14,6 +14,7 @@
 #    under the License.
 
 import collections
+import datetime as dt
 import hashlib
 import json
 
@@ -26,6 +27,61 @@ from rally.task.processing import charts
 from rally.ui import utils as ui_utils
 
 
+def _process_hooks(hooks):
+    """Prepare hooks data for report."""
+    hooks_ctx = []
+    for hook in hooks:
+        hook_ctx = {"name": hook["config"]["name"],
+                    "desc": hook["config"]["description"],
+                    "additive": [], "complete": []}
+
+        for res in hook["results"]:
+            started_at = dt.datetime.utcfromtimestamp(res["started_at"])
+            finished_at = dt.datetime.utcfromtimestamp(res["finished_at"])
+            triggered_by = "%(event_type)s: %(value)s" % res["triggered_by"]
+
+            for i, data in enumerate(res.get("output", {}).get("additive")):
+                try:
+                    hook_ctx["additive"][i]
+                except IndexError:
+                    chart_cls = plugin.Plugin.get(data["chart_plugin"])
+                    hook_ctx["additive"].append([chart_cls])
+                hook_ctx["additive"][i].append(data)
+
+            complete_charts = []
+            for data in res.get("output", {}).get("complete"):
+                chart_cls = plugin.Plugin.get(data.pop("chart_plugin"))
+                data["widget"] = chart_cls.widget
+                complete_charts.append(data)
+
+            if complete_charts:
+                hook_ctx["complete"].append(
+                    {"triggered_by": triggered_by,
+                     "started_at": started_at.strftime("%Y-%m-%d %H:%M:%S"),
+                     "finished_at": finished_at.strftime("%Y-%m-%d %H:%M:%S"),
+                     "status": res["status"],
+                     "charts": complete_charts})
+
+        for i in range(len(hook_ctx["additive"])):
+            chart_cls = hook_ctx["additive"][i].pop(0)
+            iters_count = len(hook_ctx["additive"][i])
+            first = hook_ctx["additive"][i][0]
+            descr = first.get("description", "")
+            axis_label = first.get("axis_label", "")
+            chart = chart_cls({"iterations_count": iters_count},
+                              title=first["title"],
+                              description=descr,
+                              label=first.get("label", ""),
+                              axis_label=axis_label)
+            for data in hook_ctx["additive"][i]:
+                chart.add_iteration(data["data"])
+            hook_ctx["additive"][i] = chart.render()
+
+        if hook_ctx["additive"] or hook_ctx["complete"]:
+            hooks_ctx.append(hook_ctx)
+    return hooks_ctx
+
+
 def _process_scenario(data, pos):
     main_area = charts.MainStackedAreaChart(data["info"])
     main_hist = charts.MainHistogramChart(data["info"])
@@ -75,6 +131,7 @@ def _process_scenario(data, pos):
     cls, method = data["key"]["name"].split(".")
     additive_output = [chart.render() for chart in additive_output_charts]
     iterations_count = data["info"]["iterations_count"]
+
     return {
         "cls": cls,
         "met": method,
@@ -82,6 +139,7 @@ def _process_scenario(data, pos):
         "name": method + (pos and " [%d]" % (pos + 1) or ""),
         "runner": kw["runner"]["type"],
         "config": json.dumps({data["key"]["name"]: [kw]}, indent=2),
+        "hooks": _process_hooks(data["hooks"]),
         "iterations": {
             "iter": main_area.render(),
             "pie": [("success", (data["info"]["iterations_count"]
@@ -95,6 +153,7 @@ def _process_scenario(data, pos):
         "table": main_stat.render(),
         "additive_output": additive_output,
         "complete_output": complete_output,
+        "has_output": any(additive_output) or any(complete_output),
         "output_errors": output_errors,
         "errors": errors,
         "load_duration": data["info"]["load_duration"],

rally/ui/templates/task/report.html

@@ -121,7 +121,11 @@
         },{
           id: "output",
           name: "Scenario Data",
-          visible: function(){ return $scope.scenario.output.length }
+          visible: function(){ return $scope.scenario.has_output }
+        },{
+          id: "hooks",
+          name: "Hooks",
+          visible: function(){ return $scope.scenario.hooks.length }
         },{
           id: "failures",
           name: "Failures",
@@ -138,22 +142,49 @@
 
       $scope.showTab = function(uri) {
         $scope.tab = uri.hash in $scope.tabs_map ? uri.hash : "overview";
-        if (! $scope.scenario.output) {
-          var has_additive = !! $scope.scenario.additive_output.length;
-          var has_complete = !! ($scope.scenario.complete_output.length
-                                 && $scope.scenario.complete_output[0].length);
-          $scope.scenario.output = {
-            has_additive: has_additive,
-            has_complete: has_complete,
-            length: has_additive + has_complete,
-            active: has_additive ? "additive" : (has_complete ? "complete" : "")
-          }
-        }
         if (uri.hash === "output") {
+          if (typeof $scope.scenario.output === "undefined") {
+            var has_additive = !! $scope.scenario.additive_output.length;
+            var has_complete = !! ($scope.scenario.complete_output.length
+                                   && $scope.scenario.complete_output[0].length);
+            $scope.scenario.output = {
+              has_additive: has_additive,
+              has_complete: has_complete,
+              length: has_additive + has_complete,
+              active: has_additive ? "additive" : (has_complete ? "complete" : "")
+            }
+          }
           if (uri.sub && $scope.scenario.output["has_" + uri.sub]) {
             $scope.scenario.output.active = uri.sub
           }
         }
+        else if (uri.hash === "hooks") {
+          if ($scope.scenario.hooks.length) {
+            var hook_idx = parseInt(uri.sub);
+
+            if (isNaN(hook_idx) || ($scope.scenario.hooks.length - hook_idx) <= 0) {
+              hook_idx = 0
+            }
+
+            if ($scope.scenario.hook_idx === hook_idx) {
+              return
+            }
+
+            $scope.scenario.hooks.cur = $scope.scenario.hooks[hook_idx];
+            $scope.scenario.hook_idx = hook_idx;
+            if (typeof $scope.scenario.hooks.cur.active === "undefined") {
+              if ($scope.scenario.hooks.cur.additive.length) {
+                $scope.scenario.hooks.cur.active = "additive"
+              }
+              if ($scope.scenario.hooks.cur.complete.length) {
+                if (typeof $scope.scenario.hooks.cur.active === "undefined") {
+                  $scope.scenario.hooks.cur.active = "complete"
+                }
+                $scope.set_hook_run()
+              }
+            }
+          }
+        }
       }
 
       for (var i in $scope.tabs) {
@@ -178,6 +209,23 @@
         }
       }
 
+      $scope.set_hook_run = function(idx) {
+        if (typeof idx !== "undefined") {
+          $scope.scenario.hooks.cur.run_idx = idx
+        }
+        else if (typeof $scope.scenario.hooks.cur.run_idx === "undefined") {
+          $scope.scenario.hooks.cur.run_idx = 0
+        }
+        idx = $scope.scenario.hooks.cur.run_idx;
+        if (($scope.scenario.hooks.cur.complete.length - idx) > 0) {
+          $scope.scenario.hooks.cur.run = $scope.scenario.hooks.cur.complete[idx]
+        }
+      }
+
+      $scope.complete_hooks_as_dropdown = function() {
+        return $scope.scenario.hooks.cur.complete.length > 10
+      }
+
       /* Other helpers */
 
       $scope.showError = function(message) {
@@ -262,6 +310,10 @@
     .navmet:hover { background:#f8f8f8 }
     .navmet.active, .navmet.active:hover { background:#428bca; background-image:linear-gradient(to bottom, #428bca 0px, #3278b3 100%); border-color:#3278b3; color:#fff }
 
+    .buttn { color:#555; background:#fff; border:1px solid #ddd; border-radius:5px; font-size:12px; margin-bottom:-1px; padding:5px 7px; text-align:left; text-overflow:ellipsis; white-space:nowrap; overflow:hidden; cursor:pointer }
+    .buttn:hover { background:#f8f8f8 }
+    .buttn.active, .bttn.active:hover { background:#428bca; background-image:linear-gradient(to bottom, #428bca 0px, #3278b3 100%); border-color:#3278b3; color:#fff; cursor:default }
+
     .tabs { list-style:outside none none; margin:0 0 5px; padding:0; border-bottom:1px solid #ddd }
     .tabs:after { clear:both }
     .tabs li { float:left; margin-bottom:-1px; display:block; position:relative }
@@ -272,7 +324,7 @@
     .failure-trace { color:#333; white-space:pre; overflow:auto }
 
     .link { color:#428BCA; padding:5px 15px 5px 5px; text-decoration:underline; cursor:pointer }
-    .link.active { color:#333; text-decoration:none }
+    .link.active { color:#333; text-decoration:none; cursor:default }
 
     .chart { padding:0; margin:0; width:890px }
     .chart svg { height:300px; padding:0; margin:0; overflow:visible; float:right }
@@ -380,6 +432,13 @@
                   <b ng-show="ov_srt=='errors.length' && !ov_dir">&#x25b4;</b>
                   <b ng-show="ov_srt=='errors.length' && ov_dir">&#x25be;</b>
                 </span>
+              <th class="sortable" title="Number of hooks"
+                  ng-click="ov_srt='hooks.count'; ov_dir=!ov_dir">
+                Hooks
+                <span class="arrow">
+                  <b ng-show="ov_srt=='hooks.length' && !ov_dir">&#x25b4;</b>
+                  <b ng-show="ov_srt=='hooks.length' && ov_dir">&#x25be;</b>
+                </span>
               <th class="sortable" title="Whether SLA check is successful"
                   ng-click="ov_srt='sla_success'; ov_dir=!ov_dir">
                 Success (SLA)
@@ -398,6 +457,7 @@
               <td>{{sc.iterations_count}}
               <td>{{sc.runner}}
               <td>{{sc.errors.length}}
+              <td>{{sc.hooks.length}}
               <td>
                 <span ng-show="sc.sla_success" class="status-pass">&#x2714;</span>
                 <span ng-hide="sc.sla_success" class="status-fail">&#x2716;</span>
@@ -577,6 +637,105 @@
           </div>
         </script>
 
+        <script type="text/ng-template" id="hooks">
+
+          <div style="padding:15px 0">
+            <span ng-repeat="h in scenario.hooks track by $index"
+                  class="buttn"
+                  title="{{h.desc}}"
+                  style="margin:0 10px 0 0"
+                  ng-click="location.hash('hooks/'+$index)"
+                  ng-class="{active:scenario.hook_idx===$index}">
+              {{h.name}}
+            </span>
+          </div>
+
+          <table class="striped" style="margin:5px 0 25px">
+            <thead>
+              <tr>
+                <th>Plugin
+                <th>Description
+            </thead>
+            <tbody>
+              <tr>
+                <td>{{scenario.hooks.cur.name}}
+                <td>{{scenario.hooks.cur.desc}}
+            </tbody>
+          </table>
+
+          <div>
+            <span class="link"
+                  ng-click="scenario.hooks.cur.active = 'additive'"
+                  ng-class="{active:scenario.hooks.cur.active === 'additive'}"
+                  ng-if="scenario.hooks.cur.additive.length">Aggregated</span>
+            <span class="link"
+                  ng-click="scenario.hooks.cur.active = 'complete'"
+                  ng-class="{active:scenario.hooks.cur.active === 'complete'}"
+                  ng-if="scenario.hooks.cur.complete.length">Per hook run</span>
+          </div>
+
+          <div ng-repeat="chart in scenario.hooks.cur.additive"
+               ng-if="scenario.hooks.cur.active === 'additive'">
+            <div widget="{{chart.widget}}"
+                 title="{{chart.title}}"
+                 description="{{chart.description}}"
+                 name-x="{{chart.axis_label}}"
+                 name-y="{{chart.label}}"
+                 data="chart.data">
+            </div>
+          </div>
+
+          <div ng-if="scenario.hooks.cur.active === 'complete'" style="padding:10px 0 0">
+
+            <select ng-if="complete_hooks_as_dropdown()"
+                    ng-model="scenario.hooks.cur.run_idx"
+                    ng-change="set_hook_run()">
+              <option ng-repeat="h in scenario.hooks.cur.complete track by $index"
+                      ng-selected="scenario.hooks.cur.run_idx == $index"
+                      value="{{$index}}">
+                {{h.triggered_by}}
+            </select>
+
+            <div ng-if="! complete_hooks_as_dropdown()"
+                 style="border:#ccc solid; border-width:1px 0 0; padding:5px 0 0">
+              <span ng-repeat="h in scenario.hooks.cur.complete track by $index"
+                    class="link"
+                    ng-class="{active:scenario.hooks.cur.run_idx == $index}"
+                    ng-click="set_hook_run($index)">
+                {{h.triggered_by}}
+              </span>
+            </div>
+
+            <table class="striped" style="margin:15px 0 15px">
+              <thead>
+                <tr>
+                  <th>Status
+                  <th>Triggered by
+                  <th>Started at
+                  <th>Finished at
+              </thead>
+              <tbody>
+                <tr>
+                  <td ng-style="scenario.hooks.cur.run.status === 'success' ? {color:'green'} : {color:'red'}">
+                    <b>{{scenario.hooks.cur.run.status}}</b>
+                  <td>{{scenario.hooks.cur.run.triggered_by}}
+                  <td>{{scenario.hooks.cur.run.started_at}}
+                  <td>{{scenario.hooks.cur.run.finished_at}}
+              </tbody>
+            </table>
+
+            <div ng-repeat="chart in scenario.hooks.cur.run.charts">
+              <div widget="{{chart.widget}}"
+                   title="{{chart.title}}"
+                   description="{{chart.description}}"
+                   name-x="{{chart.axis_label}}"
+                   name-y="{{chart.label}}"
+                   data="chart.data">
+              </div>
+            </div>
+          </div>
+        </script>
+
         <script type="text/ng-template" id="failures">
           <h2>Task failures (<ng-pluralize
             count="scenario.errors.length"

tests/functional/test_cli_task.py

@@ -1072,23 +1072,31 @@ class HookTestCase(unittest.TestCase):
             ]
         }
 
-    def _get_result(self, config, iterations=None, seconds=None):
+    def _get_result(self, config, iterations=None, seconds=None, error=False):
-        result = {
+        result = {"config": config, "results": [], "summary": {}}
-            "config": config,
-            "results": [],
-            "summary": {"success": 0}
-        }
         events = iterations if iterations else seconds
         event_type = "iteration" if iterations else "time"
-
+        status = "failed" if error else "success"
         for i in range(len(events)):
-            result["results"].append({
+            itr_result = {
                 "finished_at": mock.ANY,
                 "started_at": mock.ANY,
                 "triggered_by": {"event_type": event_type, "value": events[i]},
-                "status": "success"})
+                "status": status,
-            result["summary"]["success"] += 1
+                "output": {
-
+                    "additive": [],
+                    "complete": [{"chart_plugin": "TextArea",
+                                  "data": ["RetCode: %i" % error,
+                                           "StdOut: (empty)",
+                                           "StdErr: (empty)"],
+                                  "description": "Args: %s" % config["args"],
+                                  "title": "System call"}]}}
+            if error:
+                itr_result["error"] = {"etype": "n/a",
+                                       "msg": "Subprocess returned 1",
+                                       "details": "stdout: "}
+            result["results"].append(itr_result)
+        result["summary"][status] = len(events)
         return result
 
     def test_hook_result_with_constant_runner(self):
@@ -1163,12 +1171,7 @@ class HookTestCase(unittest.TestCase):
         results = json.loads(rally("task results"))
         hook_results = results[0]["hooks"]
         hooks_cfg = cfg["Dummy.dummy"][0]["hooks"]
-        expected = [self._get_result(hooks_cfg[0], iterations=[5])]
+        expected = [self._get_result(hooks_cfg[0], iterations=[5], error=True)]
-        expected[0]["results"][0]["status"] = "failed"
-        expected[0]["summary"] = {"failed": 1}
-        expected[0]["results"][0]["error"] = {"etype": "n/a",
-                                              "msg": "Subprocess returned 1",
-                                              "details": ""}
         self.assertEqual(expected, hook_results)
         self._assert_results_time(hook_results)
 

tests/unit/plugins/common/hook/test_sys_call.py

@@ -15,6 +15,7 @@
 
 import subprocess
 
+import ddt
 import jsonschema
 import mock
 
@@ -24,6 +25,7 @@ from tests.unit import fakes
 from tests.unit import test
 
 
+@ddt.ddt
 class SysCallHookTestCase(test.TestCase):
 
     def test_validate(self):
@@ -60,36 +62,51 @@ class SysCallHookTestCase(test.TestCase):
         self.assertRaises(
             jsonschema.ValidationError, sys_call.SysCallHook.validate, conf)
 
+    @ddt.data(
+        {"stdout": "foo output",
+         "expected": {
+             "additive": [],
+             "complete": [{"chart_plugin": "TextArea",
+                           "data": ["RetCode: 0", "StdOut: foo output",
+                                    "StdErr: (empty)"],
+                           "description": "Args: foo cmd",
+                           "title": "System call"}]}},
+        {"stdout": """{"additive": [],
+                       "complete": [
+                         {"chart_plugin": "Pie", "title": "Bar Pie",
+                          "data": [["A", 4], ["B", 2]]}]}""",
+         "expected": {
+             "additive": [],
+             "complete": [{"chart_plugin": "Pie", "data": [["A", 4], ["B", 2]],
+                           "title": "Bar Pie"}]}})
+    @ddt.unpack
     @mock.patch("rally.common.utils.Timer", side_effect=fakes.FakeTimer)
     @mock.patch("rally.plugins.common.hook.sys_call.subprocess.Popen")
-    def test_run(self, mock_popen, mock_timer):
+    def test_run(self, mock_popen, mock_timer, stdout, expected):
         popen_instance = mock_popen.return_value
         popen_instance.returncode = 0
+        popen_instance.communicate.return_value = (stdout, "")
+        hook = sys_call.SysCallHook(mock.Mock(), "foo cmd", {"iteration": 1})
 
-        task = mock.MagicMock()
+        hook.run_sync()
-        sys_call_hook = sys_call.SysCallHook(task, "/bin/bash -c 'ls'",
-                                             {"iteration": 1})
-
-        sys_call_hook.run_sync()
 
         self.assertEqual(
-            {
+            {"finished_at": fakes.FakeTimer().finish_timestamp(),
-                "triggered_by": {"iteration": 1},
+             "output": expected,
-                "started_at": fakes.FakeTimer().timestamp(),
+             "started_at": fakes.FakeTimer().timestamp(),
-                "finished_at": fakes.FakeTimer().finish_timestamp(),
+             "status": consts.HookStatus.SUCCESS,
-                "status": consts.HookStatus.SUCCESS,
+             "triggered_by": {"iteration": 1}},
-                "output": mock_popen.return_value.stdout.read().decode()
+            hook.result())
-            }, sys_call_hook.result())
 
-        mock_popen.assert_called_once_with(
+        mock_popen.assert_called_once_with(["foo", "cmd"],
-            ["/bin/bash", "-c", "ls"],
+                                           stdout=subprocess.PIPE,
-            stdout=subprocess.PIPE,
+                                           stderr=subprocess.PIPE)
-            stderr=subprocess.STDOUT)
 
     @mock.patch("rally.common.utils.Timer", side_effect=fakes.FakeTimer)
     @mock.patch("rally.plugins.common.hook.sys_call.subprocess.Popen")
     def test_run_error(self, mock_popen, mock_timer):
         popen_instance = mock_popen.return_value
+        popen_instance.communicate.return_value = ("foo out", "foo err")
         popen_instance.returncode = 1
         popen_instance.stdout.read.return_value = b"No such file or directory"
 
@@ -100,19 +117,23 @@ class SysCallHookTestCase(test.TestCase):
         sys_call_hook.run_sync()
 
         self.assertEqual(
-            {
+            {"error": {"details": "foo err",
-                "triggered_by": {"iteration": 1},
+                       "etype": "n/a",
-                "started_at": fakes.FakeTimer().timestamp(),
+                       "msg": "Subprocess returned 1"},
-                "finished_at": fakes.FakeTimer().finish_timestamp(),
+             "finished_at": fakes.FakeTimer().finish_timestamp(),
-                "status": consts.HookStatus.FAILED,
+             "output": {
-                "error": {
+                 "additive": [],
-                    "etype": "n/a",
+                 "complete": [{"chart_plugin": "TextArea",
-                    "msg": "Subprocess returned 1",
+                               "data": ["RetCode: 1",
-                    "details": "No such file or directory",
+                                        "StdOut: foo out",
-                }
+                                        "StdErr: foo err"],
-            }, sys_call_hook.result())
+                               "description": "Args: /bin/bash -c 'ls'",
+                               "title": "System call"}]},
+             "started_at": fakes.FakeTimer().timestamp(),
+             "status": "failed",
+             "triggered_by": {"iteration": 1}}, sys_call_hook.result())
 
         mock_popen.assert_called_once_with(
             ["/bin/bash", "-c", "ls"],
             stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT)
+            stderr=subprocess.PIPE)

tests/unit/task/processing/test_plot.py

@@ -54,27 +54,107 @@ class PlotTestCase(test.TestCase):
                          "iterations_count": 10, "iterations_passed": 10,
                          "max_duration": 14, "min_duration": 5,
                          "output_names": [],
-                         "tstamp_end": 25, "tstamp_start": 2}}
+                         "tstamp_end": 25, "tstamp_start": 2},
+                "hooks": []}
 
-        task_data = plot._process_scenario(data, 1)
+        result = plot._process_scenario(data, 1)
         self.assertEqual(
-            task_data, {
+            {"cls": "Foo", "met": "bar", "name": "bar [2]", "pos": "1",
-                "cls": "Foo", "met": "bar", "name": "bar [2]", "pos": "1",
+             "runner": "constant", "config": json.dumps(
-                "runner": "constant", "config": json.dumps(
+                 {"Foo.bar": [{"runner": {"type": "constant"}}]},
-                    {"Foo.bar": [{"runner": {"type": "constant"}}]},
+                 indent=2),
-                    indent=2),
+             "full_duration": 40, "load_duration": 32, "hooks": [],
-                "full_duration": 40, "load_duration": 32,
+             "atomic": {"histogram": "atomic_histogram",
-                "atomic": {"histogram": "atomic_histogram",
+                        "iter": "atomic_stacked", "pie": "atomic_avg"},
-                           "iter": "atomic_stacked", "pie": "atomic_avg"},
+             "iterations": {"histogram": "main_histogram",
-                "iterations": {"histogram": "main_histogram",
+                            "iter": "main_stacked",
-                               "iter": "main_stacked",
+                            "pie": [("success", 10), ("errors", 0)]},
-                               "pie": [("success", 10), ("errors", 0)]},
+             "iterations_count": 10, "errors": [],
-                "iterations_count": 10, "errors": [],
+             "load_profile": "load_profile",
-                "load_profile": "load_profile",
+             "additive_output": [],
-                "additive_output": [],
+             "complete_output": [[], [], [], [], [], [], [], [], [], []],
-                "complete_output": [[], [], [], [], [], [], [], [], [], []],
+             "has_output": False,
-                "output_errors": [],
+             "output_errors": [],
-                "sla": [], "sla_success": True, "table": "main_stats"})
+             "sla": [], "sla_success": True, "table": "main_stats"},
+            result)
+
+    @ddt.data(
+        {"hooks": [], "expected": []},
+        {"hooks": [
+            {"config": {
+                "trigger": {"args": {"at": [2, 5], "unit": "iteration"},
+                            "name": "event"},
+                "args": "foo cmd", "description": "Foo", "name": "sys_call"},
+             "results": [
+                 {"status": "success", "finished_at": 1475589987.525735,
+                  "triggered_by": {"event_type": "iteration", "value": 2},
+                  "started_at": 1475589987.433399,
+                  "output": {
+                      "additive": [
+                          {"chart_plugin": "StatsTable", "title": "Foo table",
+                           "data": [["A", 158], ["B", 177]]}],
+                      "complete": []}},
+                 {"status": "success", "finished_at": 1475589993.457818,
+                  "triggered_by": {"event_type": "iteration", "value": 5},
+                  "started_at": 1475589993.432734,
+                  "output": {
+                      "additive": [
+                          {"chart_plugin": "StatsTable", "title": "Foo table",
+                           "data": [["A", 243], ["B", 179]]}],
+                      "complete": []}}],
+             "summary": {"success": 2}},
+            {"config": {"trigger": {"args": {"at": [1, 2, 4], "unit": "time"},
+                                    "name": "event"},
+                        "args": "bar cmd", "description": "Bar hook",
+                        "name": "sys_call"},
+             "results": [
+                 {"status": "success", "finished_at": 1475589988.437791,
+                  "triggered_by": {"event_type": "time", "value": 1},
+                  "started_at": 1475589988.434244,
+                  "output": {"additive": [],
+                             "complete": [
+                                 {"chart_plugin": "Pie", "title": "Bar Pie",
+                                  "data": [["F", 4], ["G", 2]]}]}},
+                 {"status": "success",
+                  "finished_at": 1475589989.437589,
+                  "triggered_by": {"event_type": "time", "value": 2},
+                  "started_at": 1475589989.433964,
+                  "output": {"additive": [],
+                             "complete": [
+                                 {"chart_plugin": "Pie", "title": "Bar Pie",
+                                  "data": [["F", 42], ["G", 24]]}]}}],
+             "summary": {"success": 2}}],
+         "expected": [
+             {"additive": [
+                 {"data": {"cols": ["Action", "Min (sec)", "Median (sec)",
+                                    "90%ile (sec)", "95%ile (sec)",
+                                    "Max (sec)", "Avg (sec)", "Count"],
+                           "rows": [["A", 158.0, 200.5, 234.5, 238.75, 243.0,
+                                     100.75, 2],
+                                    ["B", 177.0, 178.0, 178.8, 178.9, 179.0,
+                                     89.5, 2]]},
+                  "axis_label": "", "description": "", "label": "",
+                  "title": "Foo table", "widget": "Table"}],
+              "complete": [], "desc": "Foo", "name": "sys_call"},
+             {"additive": [],
+              "complete": [
+                  {"charts": [{"data": [["F", 4], ["G", 2]],
+                               "title": "Bar Pie", "widget": "Pie"}],
+                   "finished_at": "2016-10-04 14:06:28",
+                   "started_at": "2016-10-04 14:06:28",
+                   "status": "success",
+                   "triggered_by": "time: 1"},
+                  {"charts": [{"data": [["F", 42], ["G", 24]],
+                               "title": "Bar Pie", "widget": "Pie"}],
+                   "finished_at": "2016-10-04 14:06:29",
+                   "started_at": "2016-10-04 14:06:29",
+                   "status": "success",
+                   "triggered_by": "time: 2"}],
+              "desc": "Bar hook",
+              "name": "sys_call"}]})
+    @ddt.unpack
+    def test__process_hooks(self, hooks, expected):
+        self.assertEqual(expected, plot._process_hooks(hooks))
 
     @mock.patch(PLOT + "_process_scenario")
     @mock.patch(PLOT + "json.dumps", return_value="json_data")

tests/unit/task/test_hook.py

@@ -46,7 +46,7 @@ class DummyHook(hook.Hook):
 
         output = self.config.get("output")
         if output:
-            self.set_output(output)
+            self.add_output(**output)
 
 
 class HookExecutorTestCase(test.TestCase):
@@ -94,7 +94,7 @@ class HookExecutorTestCase(test.TestCase):
     def test_result_optional(self, mock_timer, mock__timer_method):
         hook_args = self.conf["hooks"][0]["args"]
         hook_args["error"] = ["Exception", "Description", "Traceback"]
-        hook_args["output"] = {"additive": [], "complete": []}
+        hook_args["output"] = {"additive": None, "complete": None}
 
         hook_executor = hook.HookExecutor(self.conf, self.task)
         hook_executor.on_event(event_type="iteration", value=1)