diff --git a/rally/common/objects/task.py b/rally/common/objects/task.py index 28d242dec0..25127ba725 100644 --- a/rally/common/objects/task.py +++ b/rally/common/objects/task.py @@ -16,6 +16,7 @@ import json import uuid +from rally.common import costilius from rally.common import db from rally.common.i18n import _LE from rally import consts @@ -326,7 +327,7 @@ class Task(object): min_duration = 0 max_duration = 0 iterations_failed = 0 - atomic = {} + atomic = costilius.OrderedDict() output_names = set() for itr in scenario["data"]["raw"]: diff --git a/rally/common/streaming_algorithms.py b/rally/common/streaming_algorithms.py index 68e64d6af6..5a73f3b31d 100644 --- a/rally/common/streaming_algorithms.py +++ b/rally/common/streaming_algorithms.py @@ -14,13 +14,13 @@ # under the License. import abc -import heapq import math import six from rally.common.i18n import _ from rally import exceptions +from rally.task.processing import utils @six.add_metaclass(abc.ABCMeta) @@ -128,66 +128,27 @@ class MaxComputation(StreamingAlgorithm): class PercentileComputation(StreamingAlgorithm): """Compute percentile value from a stream of numbers.""" - def __init__(self, percent): + def __init__(self, percent, length): """Init streaming computation. - :param percent: numeric percent (from 0.1 to 99.9) + :param percent: numeric percent (from 0.00..1 to 0.999..) + :param length: count of the measurements """ - if not 0 < percent < 100: + if not 0 < percent < 1: raise ValueError("Unexpected percent: %s" % percent) self._percent = percent - self._count = 0 - self._left = [] - self._right = [] - self._current_percentile = None + + self._graph_zipper = utils.GraphZipper(length, 10000) def add(self, value): - value = self._cast_to_float(value) - - if self._current_percentile and value > self._current_percentile: - heapq.heappush(self._right, value) - else: - heapq.heappush(self._left, -value) - - self._count += 1 - expected_left = int(self._percent * (self._count + 1) / 100) - - if len(self._left) > expected_left: - heapq.heappush(self._right, -heapq.heappop(self._left)) - elif len(self._left) < expected_left: - heapq.heappush(self._left, -heapq.heappop(self._right)) - - left = -self._left[0] if len(self._left) else 0 - right = self._right[0] if len(self._right) else 0 - - self._current_percentile = left + (right - left) / 2. + self._graph_zipper.add_point(value) def result(self): - if self._current_percentile is None: + results = list( + map(lambda x: x[1], self._graph_zipper.get_zipped_graph())) + if not results: raise ValueError("No values have been processed") - return self._current_percentile - - -class ProgressComputation(StreamingAlgorithm): - """Compute progress in percent.""" - - def __init__(self, base_count): - """Init streaming computation. - - :param base_count: int number for end progress (100% reached) - """ - self._base_count = int(base_count) or 1 - self._count = 0 - - def add(self, *args): - if self._count >= self._base_count: - raise RuntimeError( - "100%% progress is already reached (count of %d)" - % self._base_count) - self._count += 1 - - def result(self): - return self._count / float(self._base_count) * 100 + return utils.percentile(results, self._percent) class IncrementComputation(StreamingAlgorithm): diff --git a/rally/task/processing/charts.py b/rally/task/processing/charts.py index bc7440f621..4fe32181f2 100644 --- a/rally/task/processing/charts.py +++ b/rally/task/processing/charts.py @@ -14,6 +14,7 @@ import abc import bisect +import copy import math import six @@ -270,78 +271,65 @@ class AtomicHistogramChart(HistogramChart): return list(iteration["atomic_actions"].items()) -@six.add_metaclass(abc.ABCMeta) -class Table(Chart): - """Base class for table with processed data.""" +class MainStatsTable(Chart): - @abc.abstractmethod - def _init_columns(self): - """Initialize columns processing. + def _init_row(self, name, iterations_count): - :returns: OrderedDict( - (("str column name", ), - ...)) - """ + def round_3(stream, no_result): + if no_result: + return "n/a" + return round(stream.result(), 3) + + return [ + ("Action", name), + ("Min (sec)", streaming.MinComputation(), round_3), + ("Median (sec)", + streaming.PercentileComputation(0.5, iterations_count), round_3), + ("90%ile (sec)", + streaming.PercentileComputation(0.9, iterations_count), round_3), + ("95%ile (sec)", + streaming.PercentileComputation(0.95, iterations_count), round_3), + ("Max (sec)", streaming.MaxComputation(), round_3), + ("Avg (sec)", streaming.MeanComputation(), round_3), + ("Success", streaming.MeanComputation(), + lambda stream, no_result: "%.1f%%" % (stream.result() * 100)), + ("Count", streaming.IncrementComputation(), + lambda x, no_result: x.result()) + ] + + def __init__(self, benchmark_info, zipped_size=1000): + self.rows = list(benchmark_info["atomic"].keys()) + self.rows.append("total") + self.rows_index = dict((name, i) for i, name in enumerate(self.rows)) + self.table = [self._init_row(name, benchmark_info["iterations_count"]) + for name in self.rows] def add_iteration(self, iteration): - for name, value in self._map_iteration_values(iteration): - if name not in self._data: - self._data[name] = self._init_columns() - for column in self._data[name]: - self._data[name][column].add(value or 0) + data = copy.copy(iteration["atomic_actions"]) + data["total"] = iteration["duration"] - @abc.abstractmethod - def render(self): - """Generate table data ready for displaying. - - :returns: {"cols": [str, ...], "rows": [[numeric, ...], ...]} - """ - - -class MainStatsTable(Table): - - columns = ["Action", "Min (sec)", "Median (sec)", "90%ile (sec)", - "95%ile (sec)", "Max (sec)", "Avg (sec)", "Success", "Count"] - float_columns = ["Min (sec)", "Median (sec)", "90%ile (sec)", - "95%ile (sec)", "Max (sec)", "Avg (sec)"] - - def _init_columns(self): - return costilius.OrderedDict( - (("Min (sec)", streaming.MinComputation()), - ("Median (sec)", streaming.PercentileComputation(50)), - ("90%ile (sec)", streaming.PercentileComputation(90)), - ("95%ile (sec)", streaming.PercentileComputation(95)), - ("Max (sec)", streaming.MaxComputation()), - ("Avg (sec)", streaming.MeanComputation()), - ("Success", streaming.ProgressComputation(self.base_size)), - ("Count", streaming.IncrementComputation()))) - - def _map_iteration_values(self, iteration): - iteration = self._fix_atomic_actions(iteration) - values = list(iteration["atomic_actions"].items()) - values.append(("total", - 0 if iteration["error"] else iteration["duration"])) - return values + for name, value in data.items(): + index = self.rows_index[name] + self.table[index][-1][1].add(None) + if iteration["error"]: + self.table[index][-2][1].add(0) + else: + self.table[index][-2][1].add(1) + for elem in self.table[index][1:-2]: + elem[1].add(value) def render(self): rows = [] - total = None - for name, values in self._data.items(): - row = [name] - for column_name, column in self._data[name].items(): - if column_name == "Success": - row.append("%.1f%%" % column.result()) - else: - row.append(round(column.result(), 3)) - - # Save `total' - it must be appended last - if name.lower() == "total": - total = row - continue + for i in range(len(self.table)): + row = [self.table[i][0][1]] + # no results if all iterations failed + no_result = self.table[i][-2][1].result() == 0.0 + row.extend(x[2](x[1], no_result) for x in self.table[i][1:]) rows.append(row) - if total: - rows.append(total) + return {"cols": list(map(lambda x: x[0], self.table[0])), + "rows": rows} - return {"cols": self.columns, "rows": rows} + def _map_iteration_values(self, iteration): + pass diff --git a/tests/unit/common/test_streaming_algorithms.py b/tests/unit/common/test_streaming_algorithms.py index 894f9cbb25..c771738784 100644 --- a/tests/unit/common/test_streaming_algorithms.py +++ b/tests/unit/common/test_streaming_algorithms.py @@ -111,6 +111,9 @@ class MaxComputationTestCase(test.TestCase): @ddt.ddt class PercentileComputationTestCase(test.TestCase): + mixed1 = [0] + mixed6 = [100, 100, 0, 100, 100, 100] + mixed5 = [0, 0, 100, 0, 0] mixed16 = [55.71, 83.05, 24.12, 27, 48.36, 16.36, 96.23, 6, 16.0, 88.11, 29.52, 99.2, 79.96, 77.84, 85.45, 85.32, 7, 17.1, 3.02, 15.23] mixed50 = [51.63, 82.2, 52.52, .05, 66, 94.03, 78.6, 80.9, 51.89, 79, 1.4, @@ -122,58 +125,44 @@ class PercentileComputationTestCase(test.TestCase): range5000 = range(5000) @ddt.data( - {"stream": "mixed16", "percent": 25, "expected": 16.18}, - {"stream": "mixed16", "percent": 50, "expected": 38.94}, - {"stream": "mixed16", "percent": 90, "expected": 92.17}, - {"stream": "mixed50", "percent": 25, "expected": 23.1}, - {"stream": "mixed50", "percent": 50, "expected": 51.89}, - {"stream": "mixed50", "percent": 90, "expected": 85.265}, - {"stream": "mixed5000", "percent": 25, "expected": 25.03}, - {"stream": "mixed5000", "percent": 50, "expected": 51.89}, - {"stream": "mixed5000", "percent": 90, "expected": 85.265}, - {"stream": "range5000", "percent": 25, "expected": 1249.5}, - {"stream": "range5000", "percent": 50, "expected": 2499.5}, - {"stream": "range5000", "percent": 90, "expected": 4499.5}) + {"stream": "mixed1", "percent": 0.95, "expected": 0}, + {"stream": "mixed6", "percent": 0.5, "expected": 100}, + {"stream": "mixed5", "percent": 0.5, "expected": 0}, + {"stream": "mixed5", "percent": 0.999, "expected": 99.6}, + {"stream": "mixed5", "percent": 0.001, "expected": 0}, + {"stream": "mixed16", "percent": 0.25, "expected": 16.27}, + {"stream": "mixed16", "percent": 0.50, "expected": 38.94}, + {"stream": "mixed16", "percent": 0.90, "expected": + 88.92200000000001}, + {"stream": "mixed50", "percent": 0.25, "expected": 25.105}, + {"stream": "mixed50", "percent": 0.50, "expected": 51.89}, + {"stream": "mixed50", "percent": 0.90, "expected": + 82.81300000000002}, + {"stream": "mixed5000", "percent": 0.25, "expected": + 35.54600000000001}, + {"stream": "mixed5000", "percent": 0.50, "expected": 48.351}, + {"stream": "mixed5000", "percent": 0.90, "expected": + 66.05880000000437}, + {"stream": "range5000", "percent": 0.25, "expected": 1249.75}, + {"stream": "range5000", "percent": 0.50, "expected": 2499.5}, + {"stream": "range5000", "percent": 0.90, "expected": 4499.1}) @ddt.unpack def test_add_and_result(self, percent, stream, expected): - comp = algo.PercentileComputation(percent=percent) + comp = algo.PercentileComputation(percent=percent, length=len( + getattr(self, stream))) [comp.add(i) for i in getattr(self, stream)] self.assertEqual(expected, comp.result()) def test_add_raises(self): - comp = algo.PercentileComputation(50) + comp = algo.PercentileComputation(0.50, 100) self.assertRaises(TypeError, comp.add) - self.assertRaises(TypeError, comp.add, None) - self.assertRaises(TypeError, comp.add, "str") def test_result_raises(self): self.assertRaises(TypeError, algo.PercentileComputation) - comp = algo.PercentileComputation(50) + comp = algo.PercentileComputation(0.50, 100) self.assertRaises(ValueError, comp.result) -class ProgressComputationTestCase(test.TestCase): - - def test___init__raises(self): - self.assertRaises(TypeError, algo.ProgressComputation) - self.assertRaises(TypeError, algo.ProgressComputation, None) - self.assertRaises(ValueError, algo.ProgressComputation, "str") - - def test_add_and_result(self): - comp = algo.ProgressComputation(42) - self.assertEqual(0, comp.result()) - for expected_progress in (2.38, 4.76, 7.14, 9.52, 11.9, 14.29, - 16.67, 19.05, 21.43): - comp.add(42) - self.assertEqual(expected_progress, round(comp.result(), 2)) - - def test_add_raises(self): - comp = algo.ProgressComputation(42) - [comp.add(123) for i in range(42)] - self.assertRaises(RuntimeError, comp.add, None) - self.assertRaises(RuntimeError, comp.add, 123) - - class IncrementComputationTestCase(test.TestCase): def test_add_and_result(self): diff --git a/tests/unit/task/processing/test_charts.py b/tests/unit/task/processing/test_charts.py index 8bc75fa25c..e0f5c4a848 100644 --- a/tests/unit/task/processing/test_charts.py +++ b/tests/unit/task/processing/test_charts.py @@ -340,62 +340,96 @@ class AtomicHistogramChartTestCase(test.TestCase): self.assertEqual(expected, chart.render()) -class TableTestCase(test.TestCase): - - class Table(charts.Table): - columns = ["name", "foo", "bar"] - foo = mock.Mock() - bar = mock.Mock() - - def _init_columns(self): - return costilius.OrderedDict( - [("foo", self.foo), ("bar", self.bar)]) - - def _map_iteration_values(self, iteration): - return [("value_" + k, iteration[k]) for k in ["a", "b"]] - - def render(self): - return self._data - - def setUp(self, *args, **kwargs): - super(TableTestCase, self).setUp(*args, **kwargs) - self.bench_info = {"iterations_count": 42, - "atomic": {"a": {}, "b": {}, "c": {}}} - - def test_add_iteration_and_render(self): - self.assertRaises(TypeError, charts.Table, self.bench_info) - table = self.Table(self.bench_info) - self.assertEqual(costilius.OrderedDict(), table.render()) - [table.add_iteration({"a": i, "b": 43 - i}) for i in range(1, 43)] - self.assertEqual( - costilius.OrderedDict( - [("value_a", costilius.OrderedDict([("foo", table.foo), - ("bar", table.bar)])), - ("value_b", costilius.OrderedDict([("foo", table.foo), - ("bar", table.bar)]))]), - table.render()) +MAIN_STATS_TABLE_COLUMNS = ["Action", "Min (sec)", "Median (sec)", + "90%ile (sec)", "95%ile (sec)", "Max (sec)", + "Avg (sec)", "Success", "Count"] +def generate_iteration(duration, error, *args): + return { + "atomic_actions": costilius.OrderedDict(args), + "duration": duration, + "error": error + } + + +@ddt.ddt class MainStatsTableTestCase(test.TestCase): - def setUp(self, *args, **kwargs): - super(MainStatsTableTestCase, self).setUp(*args, **kwargs) - self.bench_info = {"iterations_count": 42, - "atomic": {"a": {}, "b": {}, "c": {}}} - self.columns = [ - "Action", "Min (sec)", "Median (sec)", "90%ile (sec)", - "95%ile (sec)", "Max (sec)", "Avg (sec)", "Success", "Count"] + @ddt.data( + { + "info": { + "iterations_count": 1, + "atomic": costilius.OrderedDict([("foo", {}), ("bar", {})]) + }, + "data": [ + generate_iteration(10.0, False, ("foo", 1.0), ("bar", 2.0)) + ], + "expected": { + "cols": MAIN_STATS_TABLE_COLUMNS, + "rows": [ + ["foo", 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, "100.0%", 1], + ["bar", 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, "100.0%", 1], + ["total", 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, "100.0%", 1], + ] + } + }, + { + "info": {"iterations_count": 2, "atomic": {"foo": {}}}, + "data": [ + generate_iteration(10.0, True, ("foo", 1.0)), + generate_iteration(10.0, True, ("foo", 2.0)) + ], + "expected": { + "cols": MAIN_STATS_TABLE_COLUMNS, + "rows": [ + ["foo", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a", "0.0%", + 2], + ["total", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a", "0.0%", + 2], + ] + } + }, + { + "info": {"iterations_count": 2, "atomic": {"foo": {}}}, + "data": [ + generate_iteration(10.0, False, ("foo", 1.0)), + generate_iteration(20.0, True, ("foo", 2.0)) + ], + "expected": { + "cols": MAIN_STATS_TABLE_COLUMNS, + "rows": [ + ["foo", 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, "50.0%", 2], + ["total", 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, "50.0%", 2] + ] + } + }, + { + "info": { + "iterations_count": 4, + "atomic": costilius.OrderedDict([("foo", {}), ("bar", {})]) + }, + "data": [ + generate_iteration(10.0, False, ("foo", 1.0), ("bar", 4.0)), + generate_iteration(20.0, False, ("foo", 2.0), ("bar", 4.0)), + generate_iteration(30.0, False, ("foo", 3.0), ("bar", 4.0)), + generate_iteration(40.0, True, ("foo", 4.0), ("bar", 4.0)) + ], + "expected": { + "cols": MAIN_STATS_TABLE_COLUMNS, + "rows": [ + ["foo", 1.0, 2.0, 2.8, 2.9, 3.0, 2.0, "75.0%", 4], + ["bar", 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, "75.0%", 4], + ["total", 10.0, 20.0, 28.0, 29.0, 30.0, 20.0, "75.0%", 4] + ] + } + } + ) + @ddt.unpack + def test_add_iteration_and_render(self, info, data, expected): - def test_add_iteration_and_render(self): - table = charts.MainStatsTable({"iterations_count": 42, - "atomic": {"foo": {}, "bar": {}}}) - [table.add_iteration( - {"atomic_actions": costilius.OrderedDict([("foo", i), - ("bar", 43 - 1)]), - "duration": i, "error": i % 40}) for i in range(1, 43)] - expected_rows = [ - ["foo", 1.0, 21.5, 38.5, 40.5, 42.0, 21.5, "100.0%", 42.0], - ["bar", 42.0, 42.0, 42.0, 42.0, 42.0, 42.0, "100.0%", 42.0], - ["total", 0.0, 0.0, 0.0, 0.0, 40.0, 0.952, "100.0%", 42.0]] - self.assertEqual({"cols": self.columns, "rows": expected_rows}, - table.render()) + table = charts.MainStatsTable(info) + for el in data: + table.add_iteration(el) + + self.assertEqual(expected, table.render())