openstack/rally
Code Issues Proposed changes

Fix bug in atomic actions

Browse Source 
Some code cleanup for the recently merged https://review.openstack.org/111977

We took the atomic actions list by mistake always from the first iteration;
we should continue taking this list from the first non-failed iteration.
Add functional testing for this case.

We also fix the missing "%" sign in the output CLI table in case there were
no successful atomic action iterations.

Co-authored-by: Oleg Anufriev <oanufriev@mirantis.com>
Change-Id: Id4095122be8dd8bffeeaeabe407ff76c0f294686
Closes-Bug: 1372739
Closes-Bug: 1372998
Closes-Bug: 1373488
Closes-Bug: 1373576
This commit is contained in:
Mikhail Dubov 2014-09-23 15:50:29 +04:00
parent 30c0624821
commit e16d5c3cec
12 changed files with 101 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
doc/samples/tasks/scenarios/dummy/dummy_random_fail_in_atomic.json Normal file
View File

@ -0,0 +1,14 @@
{
"Dummy.dummy_random_fail_in_atomic": [
{
"args": {
"exception_probability": 0.5
},
"runner": {
"type": "constant",
"times": 100,
"concurrency": 10
}
}
]
}

8
doc/samples/tasks/scenarios/dummy/dummy_random_fail_in_atomic.yaml Normal file
View File

@ -0,0 +1,8 @@
Dummy.dummy_random_fail_in_atomic:
-
args:
exception_probability: 0.5
runner:
type: "constant"
times: 100
concurrency: 10

9
rally-scenarios/rally.yaml
View File

@ -289,6 +289,15 @@
sla:
max_failure_percent: 0
Dummy.dummy_random_fail_in_atomic:
-
args:
exception_probability: 0.5
runner:
type: "constant"
times: 50
concurrency: 10
FakePlugin.testplugin:
-
runner:

19
rally/benchmark/processing/plot.py
View File

@ -39,9 +39,13 @@ def _prepare_data(data, reduce_rows=1000):
for k, v in d1.iteritems():
v[-1] = (v[-1] + d2[k]) / 2.0
atomic_action_names = (data["result"][0]["atomic_actions"].keys()
if data["result"] else [])
zero_atomic_actions = dict([(a, 0) for a in atomic_action_names])
atomic_actions = []
for row in data["result"]:
# find first non-error result to get atomic actions names
if not row["error"] and "atomic_actions" in row:
atomic_actions = row["atomic_actions"].keys()
break
zero_atomic_actions = dict([(a, 0) for a in atomic_actions])
total_durations = {"duration": [], "idle_duration": []}
atomic_durations = dict([(a, []) for a in zero_atomic_actions])
@ -147,9 +151,12 @@ def _process_atomic(result, data):
# all iteration. So we should take first non "error"
# iteration. And get in atomitc_iter list:
# [{"key": "action", "values":[]}]
stacked_area = ([{"key": a, "values": []}
for a in result["result"][0]["atomic_actions"]]
if result["result"] else [])
stacked_area = []
for row in result["result"]:
if not row["error"] and "atomic_actions" in row:
stacked_area = [{"key": a, "values": []}
for a in row["atomic_actions"]]
break
# NOTE(boris-42): pie is similiar to stacked_area, only difference is in
# structure of values. In case of $error we shouldn't put

8
rally/benchmark/processing/utils.py
View File

@ -60,12 +60,16 @@ def get_atomic_actions_data(raw_data):
:returns: dictionary containing atomic action + total duration lists
for all atomic action keys
"""
atomic_actions = raw_data[0]["atomic_actions"].keys() if raw_data else []
atomic_actions = []
for row in raw_data:
# find first non-error result to get atomic actions names
if not row["error"] and "atomic_actions" in row:
atomic_actions = row["atomic_actions"].keys()
actions_data = {}
for atomic_action in atomic_actions:
actions_data[atomic_action] = [
r["atomic_actions"][atomic_action]
for r in raw_data
if r["atomic_actions"][atomic_action] is not None]
if r["atomic_actions"].get(atomic_action) is not None]
actions_data["total"] = [r["duration"] for r in raw_data if not r["error"]]
return actions_data

3
rally/benchmark/scenarios/base.py
View File

@ -279,9 +279,6 @@ class AtomicAction(utils.Timer):
else:
name_template = name + " (%i)"
atomic_action_iteration = 2
with open("1.txt", "a") as f:
f.write("Enter\n")
f.write(str(dir(self.scenario_instance)) + "\n")
while self.scenario_instance._atomic_action_registered(
name_template % atomic_action_iteration):
atomic_action_iteration += 1

10
rally/benchmark/scenarios/dummy/dummy.py
View File

@ -76,3 +76,13 @@ class Dummy(base.Scenario):
}
err = ""
return {"data": out, "errors": err}
@base.atomic_action_timer("dummy_fail_test")
def _random_fail_emitter(self, exception_probability):
if random.random() < exception_probability:
raise KeyError("Dummy test exception")
@base.scenario()
def dummy_random_fail_in_atomic(self, exception_probability=0.5):
self._random_fail_emitter(exception_probability)
self._random_fail_emitter(exception_probability)

36
rally/cmd/commands/task.py
View File

@ -131,13 +131,18 @@ class TaskCommands(object):
:param iterations_data: print detailed results for each iteration
Prints detailed information of task.
"""
def _print_iterations_data(raw):
headers = ['iteration', "full duration"]
float_cols = ['full duration']
for i in range(0, len(raw)):
if raw[i]['atomic_actions']:
for (c, a) in enumerate(raw[i]['atomic_actions'], 1):
action = str(c) + "-" + a['action']
def _print_iterations_data(raw_data):
headers = ["iteration", "full duration"]
float_cols = ["full duration"]
atomic_actions = []
for row in raw_data:
# find first non-error result to get atomic actions names
if not row["error"] and "atomic_actions" in row:
atomic_actions = row["atomic_actions"].keys()
for row in raw_data:
if row["atomic_actions"]:
for (c, a) in enumerate(atomic_actions, 1):
action = "%(no)i. %(action)s" % {"no": c, "action": a}
headers.append(action)
float_cols.append(action)
break
@ -145,18 +150,16 @@ class TaskCommands(object):
formatters = dict(zip(float_cols,
[cliutils.pretty_float_formatter(col, 3)
for col in float_cols]))
for (c, r) in enumerate(raw, 1):
for (c, r) in enumerate(raw_data, 1):
dlist = [c]
d = []
if r['atomic_actions']:
for l in r['atomic_actions']:
d.append(l['duration'])
dlist.append(sum(d))
dlist = dlist + d
if r["atomic_actions"]:
dlist.append(r["duration"])
for action in atomic_actions:
dlist.append(r["atomic_actions"].get(action) or 0)
table_rows.append(rutils.Struct(**dict(zip(headers,
dlist))))
else:
data = dlist + ["N/A" for i in range(1, len(headers))]
data = dlist + [None for i in range(1, len(headers))]
table_rows.append(rutils.Struct(**dict(zip(headers,
data))))
common_cliutils.print_list(table_rows,
@ -227,7 +230,8 @@ class TaskCommands(object):
"%.1f%%" % (len(durations) * 100.0 / len(raw)),
len(raw)]
else:
data = [action, None, None, None, None, None, 0, len(raw)]
data = [action, None, None, None, None, None,
"0.0%", len(raw)]
table_rows.append(rutils.Struct(**dict(zip(table_cols, data))))
common_cliutils.print_list(table_rows, fields=table_cols,

11
tests/benchmark/scenarios/dummy/test_dummy.py
View File

@ -57,3 +57,14 @@ class DummyTestCase(test.TestCase):
# Since the data is generated in random,
# checking for not None
self.assertNotEqual(result['data'], None)
def test_dummy_random_fail_in_atomic(self):
scenario = dummy.Dummy()
for i in range(10):
scenario.dummy_random_fail_in_atomic(exception_probability=0)
for i in range(10):
self.assertRaises(KeyError,
scenario.dummy_random_fail_in_atomic,
{'exception_probability': 1})

2
tests_ci/rally-gate.sh
View File

@ -54,4 +54,6 @@ rally task results | python -m json.tool > rally-plot/results.json
gzip -9 rally-plot/results.json
rally task detailed > rally-plot/detailed.txt
gzip -9 rally-plot/detailed.txt
rally task detailed --iterations-data > rally-plot/detailed_with_iterations.txt
gzip -9 rally-plot/detailed_with_iterations.txt
rally task sla_check | tee rally-plot/sla.txt

1
tests_ci/rally-gate/index.html
View File

@ -34,6 +34,7 @@
<li> <a href="rally-plot/task.txt"> Rally input task </a> </li>
<li> <a href="rally-plot/results.html.gz"> HTML report with graphs </a> <code>$ rally task plot2html</code></li>
<li> <a href="rally-plot/detailed.txt.gz"> Plain text aggregated data </a> <code> $ rally task detailed </code></li>
<li> <a href="rally-plot/detailed_with_iterations.txt.gz"> Plain text aggregated data with detailed iterations </a> <code> $ rally task detailed --iterations-data </code></li>
<li> <a href="rally-plot/sla.txt"> SLA checks </a> <code> $ rally task sla_check </code></li>
<li> <a href="rally-plot/results.json.gz"> Full result in json </a> <code>$ rally task results</code></li>
</ul>

11
tests_ci/test_cli_task.py
View File

@ -24,14 +24,14 @@ class TaskTestCase(unittest.TestCase):
def _get_sample_task_config(self):
return {
"KeystoneBasic.create_and_list_users": [
"Dummy.dummy_random_fail_in_atomic": [
{
"args": {
"name_length": 10
},
"runner": {
"type": "constant",
"times": 5,
"times": 100,
"concurrency": 5
}
}
@ -50,8 +50,11 @@ class TaskTestCase(unittest.TestCase):
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
rally("task start --task %s" % config.filename)
self.assertIn("KeystoneBasic.create_and_list_users",
rally("task detailed"))
detailed = rally("task detailed")
self.assertIn("Dummy.dummy_random_fail_in_atomic", detailed)
self.assertIn("dummy_fail_test (2)", detailed)
detailed_iterations_data = rally("task detailed --iterations-data")
self.assertIn("2. dummy_fail_test (2)", detailed_iterations_data)
def test_results(self):
rally = utils.Rally()