Fix bug in atomic actions

Some code cleanup for the recently merged https://review.openstack.org/111977

We took the atomic actions list by mistake always from the first iteration;
we should continue taking this list from the first non-failed iteration.
Add functional testing for this case.

We also fix the missing "%" sign in the output CLI table in case there were
no successful atomic action iterations.

Co-authored-by: Oleg Anufriev <oanufriev@mirantis.com>
Change-Id: Id4095122be8dd8bffeeaeabe407ff76c0f294686
Closes-Bug: 1372739
Closes-Bug: 1372998
Closes-Bug: 1373488
Closes-Bug: 1373576

doc/samples/tasks/scenarios/dummy/dummy_random_fail_in_atomic.json

@@ -0,0 +1,14 @@
+{
+    "Dummy.dummy_random_fail_in_atomic": [
+        {
+            "args": {
+                "exception_probability": 0.5
+            },
+            "runner": {
+                "type": "constant",
+                "times": 100,
+                "concurrency": 10
+            }
+        }
+    ]
+}

doc/samples/tasks/scenarios/dummy/dummy_random_fail_in_atomic.yaml

@@ -0,0 +1,8 @@
+  Dummy.dummy_random_fail_in_atomic:
+    -
+      args:
+        exception_probability: 0.5
+      runner:
+        type: "constant"
+        times: 100
+        concurrency: 10

rally-scenarios/rally.yaml

@@ -289,6 +289,15 @@
       sla:
         max_failure_percent: 0
 
+  Dummy.dummy_random_fail_in_atomic:
+    -
+      args:
+        exception_probability: 0.5
+      runner:
+        type: "constant"
+        times: 50
+        concurrency: 10
+
   FakePlugin.testplugin:
     -
       runner:

rally/benchmark/processing/plot.py

@@ -39,9 +39,13 @@ def _prepare_data(data, reduce_rows=1000):
         for k, v in d1.iteritems():
             v[-1] = (v[-1] + d2[k]) / 2.0
 
-    atomic_action_names = (data["result"][0]["atomic_actions"].keys()
+    atomic_actions = []
-                           if data["result"] else [])
+    for row in data["result"]:
-    zero_atomic_actions = dict([(a, 0) for a in atomic_action_names])
+        # find first non-error result to get atomic actions names
+        if not row["error"] and "atomic_actions" in row:
+            atomic_actions = row["atomic_actions"].keys()
+            break
+    zero_atomic_actions = dict([(a, 0) for a in atomic_actions])
 
     total_durations = {"duration": [], "idle_duration": []}
     atomic_durations = dict([(a, []) for a in zero_atomic_actions])
@@ -147,9 +151,12 @@ def _process_atomic(result, data):
     #                 all iteration. So we should take first non "error"
     #                 iteration. And get in atomitc_iter list:
     #                 [{"key": "action", "values":[]}]
-    stacked_area = ([{"key": a, "values": []}
+    stacked_area = []
-                     for a in result["result"][0]["atomic_actions"]]
+    for row in result["result"]:
-                    if result["result"] else [])
+        if not row["error"] and "atomic_actions" in row:
+            stacked_area = [{"key": a, "values": []}
+                            for a in row["atomic_actions"]]
+            break
 
     # NOTE(boris-42): pie is similiar to stacked_area, only difference is in
     #                 structure of values. In case of $error we shouldn't put

rally/benchmark/processing/utils.py

@@ -60,12 +60,16 @@ def get_atomic_actions_data(raw_data):
     :returns: dictionary containing atomic action + total duration lists
               for all atomic action keys
     """
-    atomic_actions = raw_data[0]["atomic_actions"].keys() if raw_data else []
+    atomic_actions = []
+    for row in raw_data:
+        # find first non-error result to get atomic actions names
+        if not row["error"] and "atomic_actions" in row:
+            atomic_actions = row["atomic_actions"].keys()
     actions_data = {}
     for atomic_action in atomic_actions:
         actions_data[atomic_action] = [
             r["atomic_actions"][atomic_action]
             for r in raw_data
-            if r["atomic_actions"][atomic_action] is not None]
+            if r["atomic_actions"].get(atomic_action) is not None]
     actions_data["total"] = [r["duration"] for r in raw_data if not r["error"]]
     return actions_data

rally/benchmark/scenarios/base.py

@@ -279,9 +279,6 @@ class AtomicAction(utils.Timer):
         else:
             name_template = name + " (%i)"
             atomic_action_iteration = 2
-            with open("1.txt", "a") as f:
-                f.write("Enter\n")
-                f.write(str(dir(self.scenario_instance)) + "\n")
             while self.scenario_instance._atomic_action_registered(
                                     name_template % atomic_action_iteration):
                 atomic_action_iteration += 1

rally/benchmark/scenarios/dummy/dummy.py

@@ -76,3 +76,13 @@ class Dummy(base.Scenario):
         }
         err = ""
         return {"data": out, "errors": err}
+
+    @base.atomic_action_timer("dummy_fail_test")
+    def _random_fail_emitter(self, exception_probability):
+        if random.random() < exception_probability:
+            raise KeyError("Dummy test exception")
+
+    @base.scenario()
+    def dummy_random_fail_in_atomic(self, exception_probability=0.5):
+        self._random_fail_emitter(exception_probability)
+        self._random_fail_emitter(exception_probability)

rally/cmd/commands/task.py

@@ -131,13 +131,18 @@ class TaskCommands(object):
         :param iterations_data: print detailed results for each iteration
         Prints detailed information of task.
         """
-        def _print_iterations_data(raw):
+        def _print_iterations_data(raw_data):
-            headers = ['iteration', "full duration"]
+            headers = ["iteration", "full duration"]
-            float_cols = ['full duration']
+            float_cols = ["full duration"]
-            for i in range(0, len(raw)):
+            atomic_actions = []
-                if raw[i]['atomic_actions']:
+            for row in raw_data:
-                    for (c, a) in enumerate(raw[i]['atomic_actions'], 1):
+                # find first non-error result to get atomic actions names
-                        action = str(c) + "-" + a['action']
+                if not row["error"] and "atomic_actions" in row:
+                    atomic_actions = row["atomic_actions"].keys()
+            for row in raw_data:
+                if row["atomic_actions"]:
+                    for (c, a) in enumerate(atomic_actions, 1):
+                        action = "%(no)i. %(action)s" % {"no": c, "action": a}
                         headers.append(action)
                         float_cols.append(action)
                     break
@@ -145,18 +150,16 @@ class TaskCommands(object):
             formatters = dict(zip(float_cols,
                                   [cliutils.pretty_float_formatter(col, 3)
                                    for col in float_cols]))
-            for (c, r) in enumerate(raw, 1):
+            for (c, r) in enumerate(raw_data, 1):
                 dlist = [c]
-                d = []
+                if r["atomic_actions"]:
-                if r['atomic_actions']:
+                    dlist.append(r["duration"])
-                    for l in r['atomic_actions']:
+                    for action in atomic_actions:
-                        d.append(l['duration'])
+                        dlist.append(r["atomic_actions"].get(action) or 0)
-                    dlist.append(sum(d))
-                    dlist = dlist + d
                     table_rows.append(rutils.Struct(**dict(zip(headers,
                                                                dlist))))
                 else:
-                    data = dlist + ["N/A" for i in range(1, len(headers))]
+                    data = dlist + [None for i in range(1, len(headers))]
                     table_rows.append(rutils.Struct(**dict(zip(headers,
                                                                data))))
             common_cliutils.print_list(table_rows,
@@ -227,7 +230,8 @@ class TaskCommands(object):
                             "%.1f%%" % (len(durations) * 100.0 / len(raw)),
                             len(raw)]
                 else:
-                    data = [action, None, None, None, None, None, 0, len(raw)]
+                    data = [action, None, None, None, None, None,
+                            "0.0%", len(raw)]
                 table_rows.append(rutils.Struct(**dict(zip(table_cols, data))))
 
             common_cliutils.print_list(table_rows, fields=table_cols,

tests/benchmark/scenarios/dummy/test_dummy.py

@@ -57,3 +57,14 @@ class DummyTestCase(test.TestCase):
         # Since the data is generated in random,
         # checking for not None
         self.assertNotEqual(result['data'], None)
+
+    def test_dummy_random_fail_in_atomic(self):
+        scenario = dummy.Dummy()
+
+        for i in range(10):
+            scenario.dummy_random_fail_in_atomic(exception_probability=0)
+
+        for i in range(10):
+            self.assertRaises(KeyError,
+                              scenario.dummy_random_fail_in_atomic,
+                              {'exception_probability': 1})

tests_ci/rally-gate.sh

@@ -54,4 +54,6 @@ rally task results | python -m json.tool > rally-plot/results.json
 gzip -9 rally-plot/results.json
 rally task detailed > rally-plot/detailed.txt
 gzip -9 rally-plot/detailed.txt
+rally task detailed --iterations-data > rally-plot/detailed_with_iterations.txt
+gzip -9 rally-plot/detailed_with_iterations.txt
 rally task sla_check | tee rally-plot/sla.txt

tests_ci/rally-gate/index.html

@@ -34,6 +34,7 @@
             <li> <a href="rally-plot/task.txt"> Rally input task </a> </li>
             <li> <a href="rally-plot/results.html.gz"> HTML report with graphs </a> <code>$ rally task plot2html</code></li>
             <li> <a href="rally-plot/detailed.txt.gz"> Plain text aggregated data </a> <code> $ rally task detailed </code></li>
+            <li> <a href="rally-plot/detailed_with_iterations.txt.gz"> Plain text aggregated data with detailed iterations </a> <code> $ rally task detailed --iterations-data </code></li>
             <li> <a href="rally-plot/sla.txt"> SLA checks  </a> <code> $ rally task sla_check </code></li>
             <li> <a href="rally-plot/results.json.gz"> Full result in json </a> <code>$ rally task results</code></li>
         </ul>

tests_ci/test_cli_task.py

@@ -24,14 +24,14 @@ class TaskTestCase(unittest.TestCase):
 
     def _get_sample_task_config(self):
         return {
-            "KeystoneBasic.create_and_list_users": [
+            "Dummy.dummy_random_fail_in_atomic": [
                 {
                     "args": {
                         "name_length": 10
                     },
                     "runner": {
                         "type": "constant",
-                        "times": 5,
+                        "times": 100,
                         "concurrency": 5
                     }
                 }
@@ -50,8 +50,11 @@ class TaskTestCase(unittest.TestCase):
         cfg = self._get_sample_task_config()
         config = utils.TaskConfig(cfg)
         rally("task start --task %s" % config.filename)
-        self.assertIn("KeystoneBasic.create_and_list_users",
+        detailed = rally("task detailed")
-                      rally("task detailed"))
+        self.assertIn("Dummy.dummy_random_fail_in_atomic", detailed)
+        self.assertIn("dummy_fail_test (2)", detailed)
+        detailed_iterations_data = rally("task detailed --iterations-data")
+        self.assertIn("2. dummy_fail_test (2)", detailed_iterations_data)
 
     def test_results(self):
         rally = utils.Rally()