From f501f397cc20738e2a2ac425f2da26b3d801a5f8 Mon Sep 17 00:00:00 2001 From: Alexander Maretskiy Date: Thu, 17 Mar 2016 18:01:31 +0200 Subject: [PATCH] [Reports] Introduce class processing.plot.Trends This class is for processing tasks results and generating data for upcoming trends report. It will be actually in use in further patches. Blueprint: trends-report Change-Id: I6f7d028b0f64d4d5ff6d6c3b7684f16a0344b169 --- rally/task/processing/plot.py | 99 ++++++++++++++++++ tests/unit/task/processing/test_plot.py | 131 ++++++++++++++++++++++++ 2 files changed, 230 insertions(+) diff --git a/rally/task/processing/plot.py b/rally/task/processing/plot.py index be78b6e3b1..4f368fee12 100644 --- a/rally/task/processing/plot.py +++ b/rally/task/processing/plot.py @@ -14,8 +14,11 @@ # under the License. import collections +import hashlib import json +import six + from rally.common import objects from rally.common.plugin import plugin from rally.task.processing import charts @@ -149,3 +152,99 @@ def plot(tasks_results, include_libs=False): source, data = _process_tasks(extended_results) return template.render(source=json.dumps(source), data=json.dumps(data), include_libs=include_libs) + + +class Trends(object): + """Process tasks results and make trends data. + + Group tasks results by their input configuration, + calculate statistics for these groups and prepare it + for displaying in trends HTML report. + """ + + def __init__(self): + self._tasks = {} + + def _to_str(self, obj): + """Convert object into string.""" + if obj is None: + return "None" + elif isinstance(obj, six.string_types + (int, float)): + return str(obj).strip() + elif isinstance(obj, (list, tuple)): + return ",".join(sorted([self._to_str(v) for v in obj])) + elif isinstance(obj, dict): + return "|".join(sorted([":".join([self._to_str(k), + self._to_str(v)]) + for k, v in obj.items()])) + raise TypeError("Unexpected type %(type)r of object %(obj)r" + % {"obj": obj, "type": type(obj)}) + + def _make_hash(self, obj): + return hashlib.md5(self._to_str(obj).encode("utf8")).hexdigest() + + def add_result(self, result): + key = self._make_hash(result["key"]["kw"]) + if key not in self._tasks: + name = result["key"]["name"] + self._tasks[key] = {"seq": 1, + "name": name, + "cls": name.split(".")[0], + "met": name.split(".")[1], + "data": {}, + "total": None, + "atomic": [], + "stat": {}, + "sla_failures": 0, + "config": json.dumps(result["key"]["kw"], + indent=2)} + else: + self._tasks[key]["seq"] += 1 + + for sla in result["sla"]: + self._tasks[key]["sla_failures"] += not sla["success"] + + task = {row[0]: dict(zip(result["info"]["stat"]["cols"], row)) + for row in result["info"]["stat"]["rows"]} + + for k in task: + for tgt, src in (("min", "Min (sec)"), + ("median", "Median (sec)"), + ("90%ile", "90%ile (sec)"), + ("95%ile", "95%ile (sec)"), + ("max", "Max (sec)"), + ("avg", "Avg (sec)")): + + # NOTE(amaretskiy): some atomic actions can be + # missed due to failures. We can ignore that + # because we use NVD3 lineChart() for displaying + # trends, which is safe for missed points + if k not in self._tasks[key]["data"]: + self._tasks[key]["data"][k] = {"min": [], + "median": [], + "90%ile": [], + "95%ile": [], + "max": [], + "avg": []} + self._tasks[key]["data"][k][tgt].append( + (self._tasks[key]["seq"], task[k][src])) + + def get_data(self): + for key, value in self._tasks.items(): + total = None + for k, v in value["data"].items(): + if k == "total": + total = v + else: + self._tasks[key]["atomic"].append( + {"name": k, "values": list(v.items())}) + for stat, comp in (("min", charts.streaming.MinComputation()), + ("max", charts.streaming.MaxComputation()), + ("avg", charts.streaming.MeanComputation())): + for k, v in total[stat]: + comp.add(v) + self._tasks[key]["stat"][stat] = comp.result() + del self._tasks[key]["data"] + self._tasks[key]["total"] = list(total.items()) + self._tasks[key]["single"] = self._tasks[key]["seq"] < 2 + return sorted(self._tasks.values(), key=lambda s: s["name"]) diff --git a/tests/unit/task/processing/test_plot.py b/tests/unit/task/processing/test_plot.py index 4728a57f0a..ba0c9b0873 100644 --- a/tests/unit/task/processing/test_plot.py +++ b/tests/unit/task/processing/test_plot.py @@ -149,3 +149,134 @@ class PlotTestCase(test.TestCase): def test__extend_results_empty(self): self.assertEqual([], plot._extend_results([])) + + +@ddt.ddt +class TrendsTestCase(test.TestCase): + + def test___init__(self): + trends = plot.Trends() + self.assertEqual({}, trends._tasks) + self.assertRaises(TypeError, plot.Trends, 42) + + @ddt.data({"args": [None], "result": "None"}, + {"args": [""], "result": ""}, + {"args": [" str value "], "result": "str value"}, + {"args": [" 42 "], "result": "42"}, + {"args": ["42"], "result": "42"}, + {"args": [42], "result": "42"}, + {"args": [42.00], "result": "42.0"}, + {"args": [[3.2, 1, " foo ", None]], "result": "1,3.2,None,foo"}, + {"args": [(" def", "abc", [22, 33])], "result": "22,33,abc,def"}, + {"args": [{}], "result": ""}, + {"args": [{1: 2, "a": " b c "}], "result": "1:2|a:b c"}, + {"args": [{"foo": "bar", (1, 2): [5, 4, 3]}], + "result": "1,2:3,4,5|foo:bar"}, + {"args": [1, 2], "raises": TypeError}, + {"args": [set()], "raises": TypeError}) + @ddt.unpack + def test__to_str(self, args, result=None, raises=None): + trends = plot.Trends() + if raises: + self.assertRaises(raises, trends._to_str, *args) + else: + self.assertEqual(result, trends._to_str(*args)) + + @mock.patch(PLOT + "hashlib") + def test__make_hash(self, mock_hashlib): + mock_hashlib.md5.return_value.hexdigest.return_value = "md5_digest" + trends = plot.Trends() + trends._to_str = mock.Mock() + trends._to_str.return_value.encode.return_value = "foo_str" + + self.assertEqual("md5_digest", trends._make_hash("foo_obj")) + trends._to_str.assert_called_once_with("foo_obj") + trends._to_str.return_value.encode.assert_called_once_with("utf8") + mock_hashlib.md5.assert_called_once_with("foo_str") + + def _make_result(self, salt, sla_success=True): + return { + "key": {"kw": salt + "_kw", "name": "Scenario.name_%s" % salt}, + "sla": [{"success": sla_success}], + "info": {"iterations_count": 4, + "atomic": {"a": 123, "b": 456}, + "stat": {"rows": [["a", 0.7, 0.85, 0.9, 0.87, + 1.25, 0.67, "100.0%", 4], + ["b", 0.5, 0.75, 0.85, 0.9, + 1.1, 0.58, "100.0%", 4], + ["total", 1.2, 1.55, 1.7, 1.9, + 1.5, 1.6, "100.0%", 4]], + "cols": ["Action", "Min (sec)", "Median (sec)", + "90%ile (sec)", "95%ile (sec)", + "Max (sec)", "Avg (sec)", "Success", + "Count"]}}, + "iterations": ["", "", "", ""]} + + def _sort_trends(self, trends_result): + for r_idx, res in enumerate(trends_result): + trends_result[r_idx]["total"].sort() + for a_idx, dummy in enumerate(res["atomic"]): + trends_result[r_idx]["atomic"][a_idx]["values"].sort() + return trends_result + + def test_add_result_and_get_data(self): + trends = plot.Trends() + for i in 0, 1: + trends.add_result(self._make_result(str(i))) + expected = [ + {"atomic": [ + {"name": "a", + "values": [("90%ile", [(1, 0.9)]), ("95%ile", [(1, 0.87)]), + ("avg", [(1, 0.67)]), ("max", [(1, 1.25)]), + ("median", [(1, 0.85)]), ("min", [(1, 0.7)])]}, + {"name": "b", + "values": [("90%ile", [(1, 0.85)]), ("95%ile", [(1, 0.9)]), + ("avg", [(1, 0.58)]), ("max", [(1, 1.1)]), + ("median", [(1, 0.75)]), ("min", [(1, 0.5)])]}], + "cls": "Scenario", "config": "\"0_kw\"", "met": "name_0", + "name": "Scenario.name_0", "seq": 1, "single": True, + "sla_failures": 0, "stat": {"avg": 1.6, "max": 1.5, "min": 1.2}, + "total": [("90%ile", [(1, 1.7)]), ("95%ile", [(1, 1.9)]), + ("avg", [(1, 1.6)]), ("max", [(1, 1.5)]), + ("median", [(1, 1.55)]), ("min", [(1, 1.2)])]}, + {"atomic": [ + {"name": "a", + "values": [("90%ile", [(1, 0.9)]), ("95%ile", [(1, 0.87)]), + ("avg", [(1, 0.67)]), ("max", [(1, 1.25)]), + ("median", [(1, 0.85)]), ("min", [(1, 0.7)])]}, + {"name": "b", + "values": [("90%ile", [(1, 0.85)]), ("95%ile", [(1, 0.9)]), + ("avg", [(1, 0.58)]), ("max", [(1, 1.1)]), + ("median", [(1, 0.75)]), ("min", [(1, 0.5)])]}], + "cls": "Scenario", "config": "\"1_kw\"", "met": "name_1", + "name": "Scenario.name_1", "seq": 1, "single": True, + "sla_failures": 0, "stat": {"avg": 1.6, "max": 1.5, "min": 1.2}, + "total": [("90%ile", [(1, 1.7)]), ("95%ile", [(1, 1.9)]), + ("avg", [(1, 1.6)]), ("max", [(1, 1.5)]), + ("median", [(1, 1.55)]), ("min", [(1, 1.2)])]}] + self.assertEqual(expected, self._sort_trends(trends.get_data())) + + def test_add_result_once_and_get_data(self): + trends = plot.Trends() + trends.add_result(self._make_result("foo", sla_success=False)) + expected = [ + {"atomic": [ + {"name": "a", + "values": [("90%ile", [(1, 0.9)]), ("95%ile", [(1, 0.87)]), + ("avg", [(1, 0.67)]), ("max", [(1, 1.25)]), + ("median", [(1, 0.85)]), ("min", [(1, 0.7)])]}, + {"name": "b", + "values": [("90%ile", [(1, 0.85)]), ("95%ile", [(1, 0.9)]), + ("avg", [(1, 0.58)]), ("max", [(1, 1.1)]), + ("median", [(1, 0.75)]), ("min", [(1, 0.5)])]}], + "cls": "Scenario", "config": "\"foo_kw\"", "met": "name_foo", + "name": "Scenario.name_foo", "seq": 1, "single": True, + "sla_failures": 1, "stat": {"avg": 1.6, "max": 1.5, "min": 1.2}, + "total": [("90%ile", [(1, 1.7)]), ("95%ile", [(1, 1.9)]), + ("avg", [(1, 1.6)]), ("max", [(1, 1.5)]), + ("median", [(1, 1.55)]), ("min", [(1, 1.2)])]}] + self.assertEqual(expected, self._sort_trends(trends.get_data())) + + def test_get_data_no_results_added(self): + trends = plot.Trends() + self.assertEqual([], trends.get_data())