Polish MQ HA scenarios and reports

performa/engine/report.py

@@ -158,8 +158,10 @@ def generate_info(definition_str, db, doc_folder, tag):
 
     for field_name, field_title in sorted(fields.items(), key=lambda a: a[0]):
         value = data[field_name]
+        if value is None:
+            value = '.'
         patt = ('''   *\n     - %%s\n     - %%%s\n''' %
-                ('d' if isinstance(value, int) else '.1f'))
+                ('.1f' if isinstance(value, float) else 's'))
         table += patt % (field_title, value)
 
     return table

performa/modules/omsimulator.py

@@ -121,6 +121,8 @@ def run(module):
               '--json %(client_file)s '
               '-l %(duration)s '
               '%(client_tool)s '
+              '--timeout %(timeout)s '
+              '-w %(sending_delay)s '
               '-p %(threads)s ') % params
 
     if params['mode'] == 'cast':
@@ -153,6 +155,10 @@ def run(module):
             round_trip_summary = client_data['summary']['round_trip']
             record['round_trip'] = round_trip_summary
 
+        if 'error' in client_data['summary']:
+            error_summary = client_data['summary']['error']
+            record['error'] = error_summary
+
         server_summary = server_data['summary']
         record['server'] = server_summary
 
@@ -174,6 +180,8 @@ def main():
             url=dict(required=True),
             threads=dict(type='int', default=10),
             duration=dict(type='int', default=10),
+            timeout=dict(type='int', default=5),
+            sending_delay=dict(type='float', default=-1.0),
         ))
 
     chdir(module)

performa/scenarios/mq/omsimulator-ha-call.rst

@@ -1,43 +1,58 @@
-Oslo.messaging simulator HA report
+RPC CALL fail-over test report
-----------------------------------
+------------------------------
 
-This report is result of `message_queue_performance`_ execution
+This scenario is executed with help of oslo.messaging simulator. There is
-with `Oslo.messaging Simulator`_
+one client-server pair of simulator running in single-threaded mode. The
+stats are collected from both client and server and detailed report is shown
+with one second precision.
 
 
-RPC CALL fail-over throughput test
+Execution Summary
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^
-
-**Summary**
 
 {{'''
-    title: Execution summary
+    title: RPC CALL Execution summary
     fields:
       a1: Client sent, msg
-      b1: Server received, msg
+      a2: Server received, msg
-      b2: Client received replies, msg
+      a3: Client received replies, msg
-      b3: Loss, msg
+      a4: Loss, msg
-      c1: Avg. request latency, ms
+      a5: Errors, msg
-      c2: Max request latency, ms
+      b1: Duration, sec
-      c3: Avg. round-trip latency, ms
+      c1: Throughput, msg/sec
-      c4: Max round-trip latency, ms
+      d1: Transfer, Mb
+      d2: Bandwidth, Mb/sec
+      e1: Avg. request latency, ms
+      e2: Max request latency, ms
+      e3: Avg. round-trip latency, ms
+      e4: Max round-trip latency, ms
     collection: records
     pipeline:
     - $match: { task: omsimulator, mode: call }
     - $project:
         a1: "$client.count"
-        b1: "$server.count"
+        a2: "$server.count"
-        b2: "$round_trip.count"
+        a3: "$round_trip.count"
-        b3: { $subtract: ["$client.count", "$round_trip.count" ] }
+        a4: { $subtract: ["$client.count", "$round_trip.count" ] }
-        c1: { $multiply: ["$server.latency", 1000] }
+        a5: "$error.count"
-        c2: { $multiply: ["$server.max_latency", 1000] }
+        b1: "$client.duration"
-        c3: { $multiply: ["$round_trip.latency", 1000] }
+        c1: { $divide: ["$client.count", "$client.duration"] }
-        c4: { $multiply: ["$round_trip.max_latency", 1000] }
+        d1: { $divide: ["$client.size", 1048576] }
+        d2: { $divide: [$divide: ["$client.size", 1048576], "$client.duration"] }
+        e1: { $multiply: ["$server.latency", 1000] }
+        e2: { $multiply: ["$server.max_latency", 1000] }
+        e3: { $multiply: ["$round_trip.latency", 1000] }
+        e4: { $multiply: ["$round_trip.max_latency", 1000] }
 ''' | info
 }}
 
 
-**Message flow**
+Message flow
+^^^^^^^^^^^^
+
+This chart shows the message flow between client and server. It includes
+messages sent by the client, received by the server, replies received by
+the client and errors caught by the client.
 
 {{'''
     title: RPC CALL message flow
@@ -46,7 +61,7 @@ RPC CALL fail-over throughput test
       y1: sent, msg
       y2: received, msg
       y3: round-trip, msg
-      y4: latency, ms
+      y4: errors, msg
     chart: line
     collection: series
     pipelines:
@@ -65,29 +80,52 @@ RPC CALL fail-over throughput test
       - $project:
           x: "$timestamp"
           y3: "$count"
-          y4: { $multiply: ["$latency", 1000] }
+    -
+      - $match: { task: omsimulator, mode: call, name: error_0 }
+      - $project:
+          x: "$timestamp"
+          y4: "$count"
 ''' | chart
 }}
 
+where:
+ * ``sent`` - messages sent by the client
+ * ``received`` - messages received by the server
+ * ``round-trip`` - replies to messages received by the client
+ * ``errors`` - errors exposed and caught by the client
 
-**Messages sent by the client**
+
+Messages sent by the client
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This chart shows messages sent by client and error rate.
 
 {{'''
     title: RPC CALL sent messages
     axes:
       x: time
       y: sent, msg
+      y2: errors, msg
     chart: line
     collection: series
-    pipeline:
+    pipelines:
-    - $match: { task: omsimulator, mode: call, name: client_0 }
+    -
-    - $project:
+      - $match: { task: omsimulator, mode: call, name: client_0 }
-        x: "$seq"
+      - $project:
-        y: "$count"
+          x: "$seq"
+          y: "$count"
+    -
+      - $match: { task: omsimulator, mode: call, name: error_0 }
+      - $project:
+          x: "$seq"
+          y2: "$count"
 ''' | chart_and_table
 }}
 
-**Messages received by the server**
+Messages received by the server
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This chart shows messages received by the server and their latency.
 
 {{'''
     title: RPC CALL received messages
@@ -106,7 +144,10 @@ RPC CALL fail-over throughput test
 ''' | chart_and_table
 }}
 
-**Round-trip messages received by the client**
+Replies received by the client
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This chart shows replies received by the client and total round-trip latency.
 
 {{'''
     title: RPC CALL round-trip messages

performa/scenarios/mq/omsimulator-ha-cast.rst

@@ -1,36 +1,51 @@
-Oslo.messaging simulator HA report
+RPC CAST fail-over test report
-----------------------------------
+------------------------------
 
-This report is result of `message_queue_performance`_ execution
+This scenario is executed with help of oslo.messaging simulator. There is
-with `Oslo.messaging Simulator`_
+one client-server pair of simulator running in single-threaded mode. The
+stats are collected from both client and server and detailed report is shown
+with one second precision.
 
 
-RPC CAST fail-over throughput test
+Execution Summary
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^
-
-**Summary**
 
 {{'''
-    title: Execution summary
+    title: RPC CAST Execution summary
     fields:
       a1: Client sent, msg
-      b1: Server received, msg
+      a2: Server received, msg
-      b2: Loss, msg
+      a3: Loss, msg
-      c1: Avg. latency, ms
+      a4: Errors, msg
-      c2: Max latency, ms
+      b1: Duration, sec
+      c1: Throughput, msg/sec
+      d1: Transfer, Mb
+      d2: Bandwidth, Mb/sec
+      e1: Avg. latency, ms
+      e2: Max latency, ms
     collection: records
     pipeline:
     - $match: { task: omsimulator, mode: cast }
     - $project:
         a1: "$client.count"
-        b1: "$server.count"
+        a2: "$server.count"
-        b2: { $subtract: ["$client.count", "$server.count" ] }
+        a3: { $subtract: ["$client.count", "$server.count" ] }
-        c1: { $multiply: ["$server.latency", 1000] }
+        a4: "$error.count"
-        c2: { $multiply: ["$server.max_latency", 1000] }
+        b1: "$client.duration"
+        c1: { $divide: ["$client.count", "$client.duration"] }
+        d1: { $divide: ["$client.size", 1048576] }
+        d2: { $divide: [$divide: ["$client.size", 1048576], "$client.duration"] }
+        e1: { $multiply: ["$server.latency", 1000] }
+        e2: { $multiply: ["$server.max_latency", 1000] }
 ''' | info
 }}
 
-**Message flow**
+Message flow
+^^^^^^^^^^^^
+
+This chart shows the message flow between client and server. It includes
+messages sent by the client, received by the server and errors caught by
+the client.
 
 {{'''
     title: RPC CAST message flow
@@ -38,7 +53,7 @@ RPC CAST fail-over throughput test
       x: time
       y1: sent, msg
       y2: received, msg
-      y3: latency, ms
+      y3: errors, msg
     chart: line
     collection: series
     pipelines:
@@ -52,29 +67,51 @@ RPC CAST fail-over throughput test
       - $project:
           x: "$timestamp"
           y2: "$count"
-          y3: { $multiply: ["$latency", 1000] }
+    -
+      - $match: { task: omsimulator, mode: cast, name: error_0 }
+      - $project:
+          x: "$timestamp"
+          y3: "$count"
 ''' | chart
 }}
 
+where:
+ * ``sent`` - messages sent by the client
+ * ``received`` - messages received by the server
+ * ``errors`` - errors exposed and caught by the client
 
-**Messages sent by the client**
+
+Messages sent by the client
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This chart shows messages sent by client and error rate.
 
 {{'''
     title: RPC CAST sent messages
     axes:
       x: time
       y: sent, msg
+      y2: errors, msg
     chart: line
     collection: series
-    pipeline:
+    pipelines:
-    - $match: { task: omsimulator, mode: cast, name: client_0 }
+    -
-    - $project:
+      - $match: { task: omsimulator, mode: cast, name: client_0 }
-        x: "$seq"
+      - $project:
-        y: "$count"
+          x: "$seq"
+          y: "$count"
+    -
+      - $match: { task: omsimulator, mode: cast, name: error_0 }
+      - $project:
+          x: "$seq"
+          y2: "$count"
 ''' | chart_and_table
 }}
 
-**Messages received by the server**
+Messages received by the server
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This chart shows messages received by the server and their latency.
 
 {{'''
     title: RPC CAST received messages

performa/scenarios/mq/omsimulator-ha-notify.rst

@@ -1,36 +1,51 @@
-Oslo.messaging simulator HA report
+NOTIFY fail-over test report
-----------------------------------
+----------------------------
 
-This report is result of `message_queue_performance`_ execution
+This scenario is executed with help of oslo.messaging simulator. There is
-with `Oslo.messaging Simulator`_
+one client-server pair of simulator running in single-threaded mode. The
+stats are collected from both client and server and detailed report is shown
+with one second precision.
 
 
-NOTIFY fail-over throughput test
+Execution Summary
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^
-
-**Summary**
 
 {{'''
-    title: Execution summary
+    title: NOTIFY Execution summary
     fields:
       a1: Client sent, msg
-      b1: Server received, msg
+      a2: Server received, msg
-      b2: Loss, msg
+      a3: Loss, msg
-      c1: Avg. latency, ms
+      a4: Errors, msg
-      c2: Max latency, ms
+      b1: Duration, sec
+      c1: Throughput, msg/sec
+      d1: Transfer, Mb
+      d2: Bandwidth, Mb/sec
+      e1: Avg. latency, ms
+      e2: Max latency, ms
     collection: records
     pipeline:
     - $match: { task: omsimulator, mode: notify }
     - $project:
         a1: "$client.count"
-        b1: "$server.count"
+        a2: "$server.count"
-        b2: { $subtract: ["$client.count", "$server.count" ] }
+        a3: { $subtract: ["$client.count", "$server.count" ] }
-        c1: { $multiply: ["$server.latency", 1000] }
+        a4: "$error.count"
-        c2: { $multiply: ["$server.max_latency", 1000] }
+        b1: "$client.duration"
+        c1: { $divide: ["$client.count", "$client.duration"] }
+        d1: { $divide: ["$client.size", 1048576] }
+        d2: { $divide: [$divide: ["$client.size", 1048576], "$client.duration"] }
+        e1: { $multiply: ["$server.latency", 1000] }
+        e2: { $multiply: ["$server.max_latency", 1000] }
 ''' | info
 }}
 
-**Message flow**
+Message flow
+^^^^^^^^^^^^
+
+This chart shows the message flow between client and server. It includes
+messages sent by the client, received by the server and errors caught by
+the client.
 
 {{'''
     title: NOTIFY message flow
@@ -38,7 +53,7 @@ NOTIFY fail-over throughput test
       x: time
       y1: sent, msg
       y2: received, msg
-      y3: latency, ms
+      y3: errors, msg
     chart: line
     collection: series
     pipelines:
@@ -52,29 +67,51 @@ NOTIFY fail-over throughput test
       - $project:
           x: "$timestamp"
           y2: "$count"
-          y3: { $multiply: ["$latency", 1000] }
+    -
+      - $match: { task: omsimulator, mode: notify, name: error_0 }
+      - $project:
+          x: "$timestamp"
+          y3: "$count"
 ''' | chart
 }}
 
+where:
+ * ``sent`` - messages sent by the client
+ * ``received`` - messages received by the server
+ * ``errors`` - errors exposed and caught by the client
 
-**Messages sent by the client**
+
+Messages sent by the client
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This chart shows messages sent by client and error rate.
 
 {{'''
     title: NOTIFY sent messages
     axes:
       x: time
       y: sent, msg
+      y2: errors, msg
     chart: line
     collection: series
-    pipeline:
+    pipelines:
-    - $match: { task: omsimulator, mode: notify, name: client_0 }
+    -
-    - $project:
+      - $match: { task: omsimulator, mode: notify, name: client_0 }
-        x: "$seq"
+      - $project:
-        y: "$count"
+          x: "$seq"
+          y: "$count"
+    -
+      - $match: { task: omsimulator, mode: notify, name: error_0 }
+      - $project:
+          x: "$seq"
+          y2: "$count"
 ''' | chart_and_table
 }}
 
-**Messages received by the server**
+Messages received by the server
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This chart shows messages received by the server and their latency.
 
 {{'''
     title: NOTIFY received messages