openstack/rally
Code Issues Proposed changes

Merge "Add atomic action names tracking"

Browse Source
This commit is contained in:
Jenkins 2014-09-22 20:54:35 +00:00 committed by Gerrit Code Review
commit 8aec519e94
21 changed files with 164 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
rally/benchmark/processing/charts/histogram.py
View File

@ -42,7 +42,7 @@ class Histogram:
def _calculate_bin_width(self):
"""Calculate the bin width using a given number of bins."""
return (self.max_data - self.min_data) / self.number_of_bins
return (self.max_data - self.min_data) // self.number_of_bins
def _calculate_x_axis(self):
"""Return a list with the values of the x axis."""

69
rally/benchmark/processing/plot.py
View File

@ -39,13 +39,9 @@ def _prepare_data(data, reduce_rows=1000):
for k, v in d1.iteritems():
v[-1] = (v[-1] + d2[k]) / 2.0
zero_atomic_actions = {}
for row in data["result"]:
# find first non-error result to get atomic actions names
if not row["error"] and "atomic_actions" in row:
zero_atomic_actions = dict([(a["action"], 0)
for a in row["atomic_actions"]])
break
atomic_action_names = (data["result"][0]["atomic_actions"].keys()
if data["result"] else [])
zero_atomic_actions = dict([(a, 0) for a in atomic_action_names])
total_durations = {"duration": [], "idle_duration": []}
atomic_durations = dict([(a, []) for a in zero_atomic_actions])
@ -74,8 +70,9 @@ def _prepare_data(data, reduce_rows=1000):
"duration": row["duration"],
"idle_duration": row["idle_duration"],
}
new_row_atomic = dict([(a["action"], a["duration"])
for a in row["atomic_actions"]])
new_row_atomic = {}
for k, v in row["atomic_actions"].iteritems():
new_row_atomic[k] = v if v else 0
if store < 1:
_append(total_durations, new_row_total)
_append(atomic_durations, new_row_atomic)
@ -136,10 +133,11 @@ def _process_atomic(result, data):
# NOTE(boris-42): In our result["result"] we have next structure:
# {"error": NoneOrDict,
# "atomic_actions": [
# {"action": String, "duration": Float},
# ...
# ]}
# "atomic_actions": {
# "action1": <duration>,
# "action2": <duration>
# }
# }
# Our goal is to get next structure:
# [{"key": $atomic_actions.action,
# "values": [[order, $atomic_actions.duration
@ -149,12 +147,9 @@ def _process_atomic(result, data):
# all iteration. So we should take first non "error"
# iteration. And get in atomitc_iter list:
# [{"key": "action", "values":[]}]
stacked_area = []
for r in result["result"]:
if not r["error"]:
for action in r["atomic_actions"]:
stacked_area.append({"key": action["action"], "values": []})
break
stacked_area = ([{"key": a, "values": []}
for a in result["result"][0]["atomic_actions"]]
if result["result"] else [])
# NOTE(boris-42): pie is similiar to stacked_area, only difference is in
# structure of values. In case of $error we shouldn't put
@ -173,9 +168,15 @@ def _process_atomic(result, data):
continue
# in case of non error put real durations to pie and stacked area
for j, action in enumerate(res["atomic_actions"]):
pie[j]["values"].append(action["duration"])
histogram_data[j]["values"].append(action["duration"])
for j, action in enumerate(res["atomic_actions"].keys()):
# in case any single atomic action failed, put 0
action_duration = res["atomic_actions"][action] or 0.0
pie[j]["values"].append(action_duration)
histogram_data[j]["values"].append(action_duration)
# filter out empty action lists in pie / histogram to avoid errors
pie = filter(lambda x: x["values"], pie)
histogram_data = filter(lambda x: x["values"], histogram_data)
histograms = [[] for atomic_action in range(len(histogram_data))]
for i, atomic_action in enumerate(histogram_data):
@ -211,28 +212,10 @@ def _process_atomic(result, data):
def _get_atomic_action_durations(result):
raw = result.get('result', [])
atomic_actions_names = []
for r in raw:
if 'atomic_actions' in r:
for a in r['atomic_actions']:
atomic_actions_names.append(a["action"])
break
action_durations = {}
for atomic_action in atomic_actions_names:
action_durations[atomic_action] = utils.get_durations(
raw,
lambda r: next(a["duration"] for a in r["atomic_actions"]
if a["action"] == atomic_action),
lambda r: any((a["action"] == atomic_action)
for a in r["atomic_actions"]))
actions_data = utils.get_atomic_actions_data(raw)
table = []
actions_list = action_durations.keys()
action_durations["total"] = utils.get_durations(
raw, lambda x: x["duration"], lambda r: not r["error"])
actions_list.append("total")
for action in actions_list:
durations = action_durations[action]
for action in actions_data:
durations = actions_data[action]
if durations:
data = [action,
round(min(durations), 3),

24
rally/benchmark/processing/utils.py
View File

@ -52,16 +52,20 @@ def percentile(values, percent):
return (d0 + d1)
def get_durations(raw_data, get_duration, is_successful):
"""Retrieve the benchmark duration data from a list of records.
def get_atomic_actions_data(raw_data):
"""Retrieve detailed (by atomic actions & total runtime) benchmark data.
:parameter raw_data: list of records
:parameter get_duration: function that retrieves the duration data from
a given record
:parameter is_successful: function that returns True if the record contains
a successful benchmark result, False otherwise
:parameter raw_data: list of raw records (scenario runner output)
:returns: list of float values corresponding to benchmark durations
:returns: dictionary containing atomic action + total duration lists
for all atomic action keys
"""
data = [get_duration(run) for run in raw_data if is_successful(run)]
return data
atomic_actions = raw_data[0]["atomic_actions"].keys() if raw_data else []
actions_data = {}
for atomic_action in atomic_actions:
actions_data[atomic_action] = [
r["atomic_actions"][atomic_action]
for r in raw_data
if r["atomic_actions"][atomic_action] is not None]
actions_data["total"] = [r["duration"] for r in raw_data if not r["error"]]
return actions_data

13
rally/benchmark/runners/base.py
View File

@ -38,7 +38,7 @@ def format_result_on_timeout(exc, timeout):
"duration": timeout,
"idle_duration": 0,
"scenario_output": {"errors": "", "data": {}},
"atomic_actions": [],
"atomic_actions": {},
"error": utils.format_exc(exc)
}
@ -117,14 +117,9 @@ class ScenarioRunnerResult(dict):
"additionalProperties": False
},
"atomic_actions": {
"type": "array",
"items": {
"type": "object",
"properties": {
"action": {"type": "string"},
"duration": {"type": "number"}
},
"additionalProperties": False
"type": "object",
"patternProperties": {
".*": {"type": ["number", "null"]}
}
},
"error": {

39
rally/benchmark/scenarios/base.py
View File

@ -55,7 +55,7 @@ class Scenario(object):
self._admin_clients = admin_clients
self._clients = clients
self._idle_duration = 0
self._atomic_actions = []
self._atomic_actions = {}
# TODO(amaretskiy): consider about prefix part of benchmark uuid
@classmethod
@ -217,10 +217,17 @@ class Scenario(object):
"""Returns duration of all sleep_between."""
return self._idle_duration
def _register_atomic_action(self, name):
"""Registers an atomic action by its name."""
self._atomic_actions[name] = None
def _atomic_action_registered(self, name):
"""Checks whether an atomic action has been already registered."""
return name in self._atomic_actions
def _add_atomic_actions(self, name, duration):
"""Adds the duration of an atomic action by its 'name'."""
self._atomic_actions.append(
{'action': name, 'duration': duration})
"""Adds the duration of an atomic action by its name."""
self._atomic_actions[name] = duration
def atomic_actions(self):
"""Returns the content of each atomic action."""
@ -236,9 +243,8 @@ def atomic_action_timer(name):
def wrap(func):
@functools.wraps(func)
def func_atomic_actions(self, *args, **kwargs):
with utils.Timer() as timer:
with AtomicAction(self, name):
f = func(self, *args, **kwargs)
self._add_atomic_actions(name, timer.duration())
return f
return func_atomic_actions
return wrap
@ -264,8 +270,25 @@ class AtomicAction(utils.Timer):
"""
super(AtomicAction, self).__init__()
self.scenario_instance = scenario_instance
self.name = name
self.name = self._get_atomic_action_name(name)
self.scenario_instance._register_atomic_action(self.name)
def _get_atomic_action_name(self, name):
if not self.scenario_instance._atomic_action_registered(name):
return name
else:
name_template = name + " (%i)"
atomic_action_iteration = 2
with open("1.txt", "a") as f:
f.write("Enter\n")
f.write(str(dir(self.scenario_instance)) + "\n")
while self.scenario_instance._atomic_action_registered(
name_template % atomic_action_iteration):
atomic_action_iteration += 1
return name_template % atomic_action_iteration
def __exit__(self, type, value, tb):
super(AtomicAction, self).__exit__(type, value, tb)
self.scenario_instance._add_atomic_actions(self.name, self.duration())
if type is None:
self.scenario_instance._add_atomic_actions(self.name,
self.duration())

27
rally/cmd/commands/task.py
View File

@ -143,23 +143,6 @@ class TaskCommands(object):
formatters=formatters)
print()
def _get_atomic_action_durations(raw):
atomic_actions_names = []
for r in raw:
if 'atomic_actions' in r:
for a in r['atomic_actions']:
atomic_actions_names.append(a["action"])
break
result = {}
for atomic_action in atomic_actions_names:
result[atomic_action] = utils.get_durations(
raw,
lambda r: next(a["duration"] for a in r["atomic_actions"]
if a["action"] == atomic_action),
lambda r: any((a["action"] == atomic_action)
for a in r["atomic_actions"]))
return result
if task_id == "last":
task = db.task_get_detailed_last()
task_id = task.uuid
@ -210,13 +193,9 @@ class TaskCommands(object):
for col in float_cols]))
table_rows = []
action_durations = _get_atomic_action_durations(raw)
actions_list = action_durations.keys()
action_durations["total"] = utils.get_durations(
raw, lambda x: x["duration"], lambda r: not r["error"])
actions_list.append("total")
for action in actions_list:
durations = action_durations[action]
actions_data = utils.get_atomic_actions_data(raw)
for action in actions_data:
durations = actions_data[action]
if durations:
data = [action,
min(durations),

41
tests/benchmark/processing/test_plot.py
View File

@ -90,17 +90,20 @@ class PlotTestCase(test.TestCase):
{
"error": [],
"duration": 1,
"idle_duration": 2
"idle_duration": 2,
"atomic_actions": {}
},
{
"error": True,
"duration": 1,
"idle_duration": 1
"idle_duration": 1,
"atomic_actions": {}
},
{
"error": [],
"duration": 2,
"idle_duration": 3
"idle_duration": 3,
"atomic_actions": {}
}
]
}
@ -153,24 +156,24 @@ class PlotTestCase(test.TestCase):
"result": [
{
"error": [],
"atomic_actions": [
{"action": "action1", "duration": 1},
{"action": "action2", "duration": 2}
]
"atomic_actions": {
"action1": 1,
"action2": 2
}
},
{
"error": ["some", "error", "occurred"],
"atomic_actions": [
{"action": "action1", "duration": 1},
{"action": "action2", "duration": 2}
]
"atomic_actions": {
"action1": 1,
"action2": 2
}
},
{
"error": [],
"atomic_actions": [
{"action": "action1", "duration": 3},
{"action": "action2", "duration": 4}
]
"atomic_actions": {
"action1": 3,
"action2": 4
}
}
]
}
@ -262,10 +265,10 @@ class PlotTestCase(test.TestCase):
data = []
for i in range(100):
atomic_actions = [
{"action": "a1", "duration": i + 0.1},
{"action": "a2", "duration": i + 0.8},
]
atomic_actions = {
"a1": i + 0.1,
"a2": i + 0.8
}
row = {
"duration": i * 3.14,
"idle_duration": i * 0.2,

42
tests/benchmark/processing/test_utils.py
View File

@ -18,7 +18,7 @@ from rally import exceptions
from tests import test
class ProcessingUtilsTestCase(test.TestCase):
class MathTestCase(test.TestCase):
def test_percentile(self):
lst = range(1, 101)
@ -43,3 +43,43 @@ class ProcessingUtilsTestCase(test.TestCase):
lst = []
self.assertRaises(exceptions.InvalidArgumentsException,
utils.mean, lst)
class AtomicActionsDataTestCase(test.TestCase):
def test_get_atomic_actions_data(self):
raw_data = [
{
"error": [],
"duration": 3,
"atomic_actions": {
"action1": 1,
"action2": 2
}
},
{
"error": ["some", "error", "occurred"],
"duration": 1.9,
"atomic_actions": {
"action1": 0.5,
"action2": 1.4
}
},
{
"error": [],
"duration": 8,
"atomic_actions": {
"action1": 4,
"action2": 4
}
}
]
atomic_actions_data = {
"action1": [1, 0.5, 4],
"action2": [2, 1.4, 4],
"total": [3, 8]
}
output = utils.get_atomic_actions_data(raw_data)
self.assertEqual(output, atomic_actions_data)

12
tests/benchmark/runners/test_base.py
View File

@ -34,7 +34,7 @@ class ScenarioHelpersTestCase(test.TestCase):
"duration": 100,
"idle_duration": 0,
"scenario_output": {"errors": "", "data": {}},
"atomic_actions": [],
"atomic_actions": {},
"error": mock_format_exc.return_value
}
@ -94,7 +94,7 @@ class ScenarioHelpersTestCase(test.TestCase):
"idle_duration": 0,
"error": [],
"scenario_output": {"errors": "", "data": {}},
"atomic_actions": []
"atomic_actions": {}
}
self.assertEqual(expected_result, result)
@ -112,7 +112,7 @@ class ScenarioHelpersTestCase(test.TestCase):
"idle_duration": 0,
"error": [],
"scenario_output": fakes.FakeScenario().with_output(),
"atomic_actions": []
"atomic_actions": {}
}
self.assertEqual(expected_result, result)
@ -128,7 +128,7 @@ class ScenarioHelpersTestCase(test.TestCase):
"duration": fakes.FakeTimer().duration(),
"idle_duration": 0,
"scenario_output": {"errors": "", "data": {}},
"atomic_actions": []
"atomic_actions": {}
}
self.assertEqual(expected_result, result)
self.assertEqual(expected_error[:2],
@ -146,7 +146,7 @@ class ScenarioRunnerResultTestCase(test.TestCase):
"data": {"test": 1.0},
"errors": "test error string 1"
},
"atomic_actions": [{"action": "test1", "duration": 1.0}],
"atomic_actions": {"test1": 1.0},
"error": []
},
{
@ -156,7 +156,7 @@ class ScenarioRunnerResultTestCase(test.TestCase):
"data": {"test": 2.0},
"errors": "test error string 2"
},
"atomic_actions": [{"action": "test2", "duration": 2.0}],
"atomic_actions": {"test2": 2.0},
"error": ["a", "b", "c"]
}
]

2
tests/benchmark/runners/test_serial.py
View File

@ -29,7 +29,7 @@ class SerialScenarioRunnerTestCase(test.TestCase):
def test_run_scenario(self, mock_run_once):
times = 5
result = {"duration": 10, "idle_duration": 0, "error": [],
"scenario_output": {}, "atomic_actions": []}
"scenario_output": {}, "atomic_actions": {}}
mock_run_once.return_value = result
expected_results = [result for i in range(times)]

4
tests/benchmark/scenarios/ceilometer/test_utils.py
View File

@ -15,7 +15,6 @@
import mock
from rally.benchmark.scenarios.ceilometer import utils
from tests.benchmark.scenarios import test_base
from tests import fakes
from tests import test
@ -30,8 +29,7 @@ class CeilometerScenarioTestCase(test.TestCase):
return_value=fakes.FakeCeilometerClient())
def _test_atomic_action_timer(self, atomic_actions_time, name):
action_duration = test_base.get_atomic_action_timer_value_by_name(
atomic_actions_time, name)
action_duration = atomic_actions_time.get(name)
self.assertIsNotNone(action_duration)
self.assertIsInstance(action_duration, float)

4
tests/benchmark/scenarios/cinder/test_utils.py
View File

@ -18,7 +18,6 @@ from oslo.config import cfg
from oslotest import mockpatch
from rally.benchmark.scenarios.cinder import utils
from tests.benchmark.scenarios import test_base
from tests import test
BM_UTILS = 'rally.benchmark.utils'
@ -43,8 +42,7 @@ class CinderScenarioTestCase(test.TestCase):
self.scenario = utils.CinderScenario()
def _test_atomic_action_timer(self, atomic_actions, name):
action_duration = test_base.get_atomic_action_timer_value_by_name(
atomic_actions, name)
action_duration = atomic_actions.get(name)
self.assertIsNotNone(action_duration)
self.assertIsInstance(action_duration, float)

4
tests/benchmark/scenarios/designate/test_utils.py
View File

@ -17,7 +17,6 @@
import mock
from rally.benchmark.scenarios.designate import utils
from tests.benchmark.scenarios import test_base
from tests import test
@ -31,8 +30,7 @@ class DesignateScenarioTestCase(test.TestCase):
self.domain = mock.Mock()
def _test_atomic_action_timer(self, atomic_actions_time, name):
action_duration = test_base.get_atomic_action_timer_value_by_name(
atomic_actions_time, name)
action_duration = atomic_actions_time.get(name)
self.assertIsNotNone(action_duration)
self.assertIsInstance(action_duration, float)

4
tests/benchmark/scenarios/glance/test_utils.py
View File

@ -18,7 +18,6 @@ from oslotest import mockpatch
from rally.benchmark.scenarios.glance import utils
from rally.benchmark import utils as butils
from rally import exceptions as rally_exceptions
from tests.benchmark.scenarios import test_base
from tests import fakes
from tests import test
@ -53,8 +52,7 @@ class GlanceScenarioTestCase(test.TestCase):
image_manager.create('fails', 'url', 'cf', 'df'))
def _test_atomic_action_timer(self, atomic_actions, name):
action_duration = test_base.get_atomic_action_timer_value_by_name(
atomic_actions, name)
action_duration = atomic_actions.get(name)
self.assertIsNotNone(action_duration)
self.assertIsInstance(action_duration, float)

4
tests/benchmark/scenarios/heat/test_utils.py
View File

@ -17,7 +17,6 @@ import mock
from oslotest import mockpatch
from rally.benchmark.scenarios.heat import utils
from tests.benchmark.scenarios import test_base
from tests import test
BM_UTILS = 'rally.benchmark.utils'
@ -43,8 +42,7 @@ class HeatScenarioTestCase(test.TestCase):
self.scenario = utils.HeatScenario()
def _test_atomic_action_timer(self, atomic_actions, name):
action_duration = test_base.get_atomic_action_timer_value_by_name(
atomic_actions, name)
action_duration = atomic_actions.get(name)
self.assertIsNotNone(action_duration)
self.assertIsInstance(action_duration, float)

4
tests/benchmark/scenarios/keystone/test_utils.py
View File

@ -16,7 +16,6 @@
import mock
from rally.benchmark.scenarios.keystone import utils
from tests.benchmark.scenarios import test_base
from tests import fakes
from tests import test
@ -52,8 +51,7 @@ class KeystoneUtilsTestCase(test.TestCase):
class KeystoneScenarioTestCase(test.TestCase):
def _test_atomic_action_timer(self, atomic_actions, name):
action_duration = test_base.get_atomic_action_timer_value_by_name(
atomic_actions, name)
action_duration = atomic_actions.get(name)
self.assertIsNotNone(action_duration)
self.assertIsInstance(action_duration, float)

4
tests/benchmark/scenarios/neutron/test_utils.py
View File

@ -17,7 +17,6 @@ import mock
import netaddr
from rally.benchmark.scenarios.neutron import utils
from tests.benchmark.scenarios import test_base
from tests import test
@ -31,8 +30,7 @@ class NeutronScenarioTestCase(test.TestCase):
self.network = mock.Mock()
def _test_atomic_action_timer(self, atomic_actions_time, name):
action_duration = test_base.get_atomic_action_timer_value_by_name(
atomic_actions_time, name)
action_duration = atomic_actions_time.get(name)
self.assertIsNotNone(action_duration)
self.assertIsInstance(action_duration, float)

4
tests/benchmark/scenarios/nova/test_utils.py
View File

@ -20,7 +20,6 @@ from oslotest import mockpatch
from rally.benchmark.scenarios.nova import utils
from rally.benchmark import utils as butils
from rally import exceptions as rally_exceptions
from tests.benchmark.scenarios import test_base
from tests import fakes
from tests import test
@ -51,8 +50,7 @@ class NovaScenarioTestCase(test.TestCase):
self.useFixture(mockpatch.Patch('time.sleep'))
def _test_atomic_action_timer(self, atomic_actions, name):
action_duration = test_base.get_atomic_action_timer_value_by_name(
atomic_actions, name)
action_duration = atomic_actions.get(name)
self.assertIsNotNone(action_duration)
self.assertIsInstance(action_duration, float)

4
tests/benchmark/scenarios/quotas/test_utils.py
View File

@ -16,7 +16,6 @@
import mock
from rally.benchmark.scenarios.quotas import utils
from tests.benchmark.scenarios import test_base
from tests import fakes
from tests import test
@ -27,8 +26,7 @@ class QuotasScenarioTestCase(test.TestCase):
super(QuotasScenarioTestCase, self).setUp()
def _test_atomic_action_timer(self, atomic_actions_time, name):
action_duration = test_base.get_atomic_action_timer_value_by_name(
atomic_actions_time, name)
action_duration = atomic_actions_time.get(name)
self.assertIsNotNone(action_duration)
self.assertIsInstance(action_duration, float)

4
tests/benchmark/scenarios/sahara/test_utils.py
View File

@ -18,7 +18,6 @@ from saharaclient.api import base as sahara_base
from rally.benchmark.scenarios.sahara import utils
from rally import exceptions
from tests.benchmark.scenarios import test_base
from tests import test
@ -28,8 +27,7 @@ SAHARA_UTILS = 'rally.benchmark.scenarios.sahara.utils'
class SaharaNodeGroupTemplatesScenarioTestCase(test.TestCase):
def _test_atomic_action_timer(self, atomic_actions, name):
action_duration = test_base.get_atomic_action_timer_value_by_name(
atomic_actions, name)
action_duration = atomic_actions.get(name)
self.assertIsNotNone(action_duration)
self.assertIsInstance(action_duration, float)

17
tests/benchmark/scenarios/test_base.py
View File

@ -330,24 +330,17 @@ class ScenarioTestCase(test.TestCase):
class AtomicActionTestCase(test.TestCase):
def test__init__(self):
fake_scenario_instance = mock.MagicMock()
fake_scenario_instance = fakes.FakeScenario()
c = base.AtomicAction(fake_scenario_instance, 'asdf')
self.assertEqual(c.scenario_instance, fake_scenario_instance)
self.assertEqual(c.name, 'asdf')
@mock.patch('tests.fakes.FakeScenario._add_atomic_actions')
@mock.patch('rally.utils.time')
def test__exit__(self, mock_time):
fake_scenario_instance = mock.Mock()
def test__exit__(self, mock_time, mock__add_atomic_actions):
fake_scenario_instance = fakes.FakeScenario()
self.start = mock_time.time()
with base.AtomicAction(fake_scenario_instance, "asdf"):
pass
duration = mock_time.time() - self.start
fake_scenario_instance._add_atomic_actions.assert_called_once_with(
'asdf', duration)
def get_atomic_action_timer_value_by_name(atomic_actions, name):
for action in atomic_actions:
if action['action'] == name:
return action['duration']
return None
mock__add_atomic_actions.assert_called_once_with('asdf', duration)