Browse Source

Polish MQ HA scenarios and reports

changes/96/298296/1
Ilya Shakhat 3 years ago
parent
commit
225fefe4db

+ 3
- 1
performa/engine/report.py View File

@@ -158,8 +158,10 @@ def generate_info(definition_str, db, doc_folder, tag):
158 158
 
159 159
     for field_name, field_title in sorted(fields.items(), key=lambda a: a[0]):
160 160
         value = data[field_name]
161
+        if value is None:
162
+            value = '.'
161 163
         patt = ('''   *\n     - %%s\n     - %%%s\n''' %
162
-                ('d' if isinstance(value, int) else '.1f'))
164
+                ('.1f' if isinstance(value, float) else 's'))
163 165
         table += patt % (field_title, value)
164 166
 
165 167
     return table

+ 8
- 0
performa/modules/omsimulator.py View File

@@ -121,6 +121,8 @@ def run(module):
121 121
               '--json %(client_file)s '
122 122
               '-l %(duration)s '
123 123
               '%(client_tool)s '
124
+              '--timeout %(timeout)s '
125
+              '-w %(sending_delay)s '
124 126
               '-p %(threads)s ') % params
125 127
 
126 128
     if params['mode'] == 'cast':
@@ -153,6 +155,10 @@ def run(module):
153 155
             round_trip_summary = client_data['summary']['round_trip']
154 156
             record['round_trip'] = round_trip_summary
155 157
 
158
+        if 'error' in client_data['summary']:
159
+            error_summary = client_data['summary']['error']
160
+            record['error'] = error_summary
161
+
156 162
         server_summary = server_data['summary']
157 163
         record['server'] = server_summary
158 164
 
@@ -174,6 +180,8 @@ def main():
174 180
             url=dict(required=True),
175 181
             threads=dict(type='int', default=10),
176 182
             duration=dict(type='int', default=10),
183
+            timeout=dict(type='int', default=5),
184
+            sending_delay=dict(type='float', default=-1.0),
177 185
         ))
178 186
 
179 187
     chdir(module)

+ 75
- 34
performa/scenarios/mq/omsimulator-ha-call.rst View File

@@ -1,43 +1,58 @@
1
-Oslo.messaging simulator HA report
2
-----------------------------------
1
+RPC CALL fail-over test report
2
+------------------------------
3 3
 
4
-This report is result of `message_queue_performance`_ execution
5
-with `Oslo.messaging Simulator`_
4
+This scenario is executed with help of oslo.messaging simulator. There is
5
+one client-server pair of simulator running in single-threaded mode. The
6
+stats are collected from both client and server and detailed report is shown
7
+with one second precision.
6 8
 
7 9
 
8
-RPC CALL fail-over throughput test
9
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
10
-
11
-**Summary**
10
+Execution Summary
11
+^^^^^^^^^^^^^^^^^
12 12
 
13 13
 {{'''
14
-    title: Execution summary
14
+    title: RPC CALL Execution summary
15 15
     fields:
16 16
       a1: Client sent, msg
17
-      b1: Server received, msg
18
-      b2: Client received replies, msg
19
-      b3: Loss, msg
20
-      c1: Avg. request latency, ms
21
-      c2: Max request latency, ms
22
-      c3: Avg. round-trip latency, ms
23
-      c4: Max round-trip latency, ms
17
+      a2: Server received, msg
18
+      a3: Client received replies, msg
19
+      a4: Loss, msg
20
+      a5: Errors, msg
21
+      b1: Duration, sec
22
+      c1: Throughput, msg/sec
23
+      d1: Transfer, Mb
24
+      d2: Bandwidth, Mb/sec
25
+      e1: Avg. request latency, ms
26
+      e2: Max request latency, ms
27
+      e3: Avg. round-trip latency, ms
28
+      e4: Max round-trip latency, ms
24 29
     collection: records
25 30
     pipeline:
26 31
     - $match: { task: omsimulator, mode: call }
27 32
     - $project:
28 33
         a1: "$client.count"
29
-        b1: "$server.count"
30
-        b2: "$round_trip.count"
31
-        b3: { $subtract: ["$client.count", "$round_trip.count" ] }
32
-        c1: { $multiply: ["$server.latency", 1000] }
33
-        c2: { $multiply: ["$server.max_latency", 1000] }
34
-        c3: { $multiply: ["$round_trip.latency", 1000] }
35
-        c4: { $multiply: ["$round_trip.max_latency", 1000] }
34
+        a2: "$server.count"
35
+        a3: "$round_trip.count"
36
+        a4: { $subtract: ["$client.count", "$round_trip.count" ] }
37
+        a5: "$error.count"
38
+        b1: "$client.duration"
39
+        c1: { $divide: ["$client.count", "$client.duration"] }
40
+        d1: { $divide: ["$client.size", 1048576] }
41
+        d2: { $divide: [$divide: ["$client.size", 1048576], "$client.duration"] }
42
+        e1: { $multiply: ["$server.latency", 1000] }
43
+        e2: { $multiply: ["$server.max_latency", 1000] }
44
+        e3: { $multiply: ["$round_trip.latency", 1000] }
45
+        e4: { $multiply: ["$round_trip.max_latency", 1000] }
36 46
 ''' | info
37 47
 }}
38 48
 
39 49
 
40
-**Message flow**
50
+Message flow
51
+^^^^^^^^^^^^
52
+
53
+This chart shows the message flow between client and server. It includes
54
+messages sent by the client, received by the server, replies received by
55
+the client and errors caught by the client.
41 56
 
42 57
 {{'''
43 58
     title: RPC CALL message flow
@@ -46,7 +61,7 @@ RPC CALL fail-over throughput test
46 61
       y1: sent, msg
47 62
       y2: received, msg
48 63
       y3: round-trip, msg
49
-      y4: latency, ms
64
+      y4: errors, msg
50 65
     chart: line
51 66
     collection: series
52 67
     pipelines:
@@ -65,29 +80,52 @@ RPC CALL fail-over throughput test
65 80
       - $project:
66 81
           x: "$timestamp"
67 82
           y3: "$count"
68
-          y4: { $multiply: ["$latency", 1000] }
83
+    -
84
+      - $match: { task: omsimulator, mode: call, name: error_0 }
85
+      - $project:
86
+          x: "$timestamp"
87
+          y4: "$count"
69 88
 ''' | chart
70 89
 }}
71 90
 
91
+where:
92
+ * ``sent`` - messages sent by the client
93
+ * ``received`` - messages received by the server
94
+ * ``round-trip`` - replies to messages received by the client
95
+ * ``errors`` - errors exposed and caught by the client
72 96
 
73
-**Messages sent by the client**
97
+
98
+Messages sent by the client
99
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
100
+
101
+This chart shows messages sent by client and error rate.
74 102
 
75 103
 {{'''
76 104
     title: RPC CALL sent messages
77 105
     axes:
78 106
       x: time
79 107
       y: sent, msg
108
+      y2: errors, msg
80 109
     chart: line
81 110
     collection: series
82
-    pipeline:
83
-    - $match: { task: omsimulator, mode: call, name: client_0 }
84
-    - $project:
85
-        x: "$seq"
86
-        y: "$count"
111
+    pipelines:
112
+    -
113
+      - $match: { task: omsimulator, mode: call, name: client_0 }
114
+      - $project:
115
+          x: "$seq"
116
+          y: "$count"
117
+    -
118
+      - $match: { task: omsimulator, mode: call, name: error_0 }
119
+      - $project:
120
+          x: "$seq"
121
+          y2: "$count"
87 122
 ''' | chart_and_table
88 123
 }}
89 124
 
90
-**Messages received by the server**
125
+Messages received by the server
126
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
127
+
128
+This chart shows messages received by the server and their latency.
91 129
 
92 130
 {{'''
93 131
     title: RPC CALL received messages
@@ -106,7 +144,10 @@ RPC CALL fail-over throughput test
106 144
 ''' | chart_and_table
107 145
 }}
108 146
 
109
-**Round-trip messages received by the client**
147
+Replies received by the client
148
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
149
+
150
+This chart shows replies received by the client and total round-trip latency.
110 151
 
111 152
 {{'''
112 153
     title: RPC CALL round-trip messages

+ 64
- 27
performa/scenarios/mq/omsimulator-ha-cast.rst View File

@@ -1,36 +1,51 @@
1
-Oslo.messaging simulator HA report
2
-----------------------------------
1
+RPC CAST fail-over test report
2
+------------------------------
3 3
 
4
-This report is result of `message_queue_performance`_ execution
5
-with `Oslo.messaging Simulator`_
4
+This scenario is executed with help of oslo.messaging simulator. There is
5
+one client-server pair of simulator running in single-threaded mode. The
6
+stats are collected from both client and server and detailed report is shown
7
+with one second precision.
6 8
 
7 9
 
8
-RPC CAST fail-over throughput test
9
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
10
-
11
-**Summary**
10
+Execution Summary
11
+^^^^^^^^^^^^^^^^^
12 12
 
13 13
 {{'''
14
-    title: Execution summary
14
+    title: RPC CAST Execution summary
15 15
     fields:
16 16
       a1: Client sent, msg
17
-      b1: Server received, msg
18
-      b2: Loss, msg
19
-      c1: Avg. latency, ms
20
-      c2: Max latency, ms
17
+      a2: Server received, msg
18
+      a3: Loss, msg
19
+      a4: Errors, msg
20
+      b1: Duration, sec
21
+      c1: Throughput, msg/sec
22
+      d1: Transfer, Mb
23
+      d2: Bandwidth, Mb/sec
24
+      e1: Avg. latency, ms
25
+      e2: Max latency, ms
21 26
     collection: records
22 27
     pipeline:
23 28
     - $match: { task: omsimulator, mode: cast }
24 29
     - $project:
25 30
         a1: "$client.count"
26
-        b1: "$server.count"
27
-        b2: { $subtract: ["$client.count", "$server.count" ] }
28
-        c1: { $multiply: ["$server.latency", 1000] }
29
-        c2: { $multiply: ["$server.max_latency", 1000] }
31
+        a2: "$server.count"
32
+        a3: { $subtract: ["$client.count", "$server.count" ] }
33
+        a4: "$error.count"
34
+        b1: "$client.duration"
35
+        c1: { $divide: ["$client.count", "$client.duration"] }
36
+        d1: { $divide: ["$client.size", 1048576] }
37
+        d2: { $divide: [$divide: ["$client.size", 1048576], "$client.duration"] }
38
+        e1: { $multiply: ["$server.latency", 1000] }
39
+        e2: { $multiply: ["$server.max_latency", 1000] }
30 40
 ''' | info
31 41
 }}
32 42
 
33
-**Message flow**
43
+Message flow
44
+^^^^^^^^^^^^
45
+
46
+This chart shows the message flow between client and server. It includes
47
+messages sent by the client, received by the server and errors caught by
48
+the client.
34 49
 
35 50
 {{'''
36 51
     title: RPC CAST message flow
@@ -38,7 +53,7 @@ RPC CAST fail-over throughput test
38 53
       x: time
39 54
       y1: sent, msg
40 55
       y2: received, msg
41
-      y3: latency, ms
56
+      y3: errors, msg
42 57
     chart: line
43 58
     collection: series
44 59
     pipelines:
@@ -52,29 +67,51 @@ RPC CAST fail-over throughput test
52 67
       - $project:
53 68
           x: "$timestamp"
54 69
           y2: "$count"
55
-          y3: { $multiply: ["$latency", 1000] }
70
+    -
71
+      - $match: { task: omsimulator, mode: cast, name: error_0 }
72
+      - $project:
73
+          x: "$timestamp"
74
+          y3: "$count"
56 75
 ''' | chart
57 76
 }}
58 77
 
78
+where:
79
+ * ``sent`` - messages sent by the client
80
+ * ``received`` - messages received by the server
81
+ * ``errors`` - errors exposed and caught by the client
59 82
 
60
-**Messages sent by the client**
83
+
84
+Messages sent by the client
85
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
86
+
87
+This chart shows messages sent by client and error rate.
61 88
 
62 89
 {{'''
63 90
     title: RPC CAST sent messages
64 91
     axes:
65 92
       x: time
66 93
       y: sent, msg
94
+      y2: errors, msg
67 95
     chart: line
68 96
     collection: series
69
-    pipeline:
70
-    - $match: { task: omsimulator, mode: cast, name: client_0 }
71
-    - $project:
72
-        x: "$seq"
73
-        y: "$count"
97
+    pipelines:
98
+    -
99
+      - $match: { task: omsimulator, mode: cast, name: client_0 }
100
+      - $project:
101
+          x: "$seq"
102
+          y: "$count"
103
+    -
104
+      - $match: { task: omsimulator, mode: cast, name: error_0 }
105
+      - $project:
106
+          x: "$seq"
107
+          y2: "$count"
74 108
 ''' | chart_and_table
75 109
 }}
76 110
 
77
-**Messages received by the server**
111
+Messages received by the server
112
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
113
+
114
+This chart shows messages received by the server and their latency.
78 115
 
79 116
 {{'''
80 117
     title: RPC CAST received messages

+ 64
- 27
performa/scenarios/mq/omsimulator-ha-notify.rst View File

@@ -1,36 +1,51 @@
1
-Oslo.messaging simulator HA report
2
-----------------------------------
1
+NOTIFY fail-over test report
2
+----------------------------
3 3
 
4
-This report is result of `message_queue_performance`_ execution
5
-with `Oslo.messaging Simulator`_
4
+This scenario is executed with help of oslo.messaging simulator. There is
5
+one client-server pair of simulator running in single-threaded mode. The
6
+stats are collected from both client and server and detailed report is shown
7
+with one second precision.
6 8
 
7 9
 
8
-NOTIFY fail-over throughput test
9
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
10
-
11
-**Summary**
10
+Execution Summary
11
+^^^^^^^^^^^^^^^^^
12 12
 
13 13
 {{'''
14
-    title: Execution summary
14
+    title: NOTIFY Execution summary
15 15
     fields:
16 16
       a1: Client sent, msg
17
-      b1: Server received, msg
18
-      b2: Loss, msg
19
-      c1: Avg. latency, ms
20
-      c2: Max latency, ms
17
+      a2: Server received, msg
18
+      a3: Loss, msg
19
+      a4: Errors, msg
20
+      b1: Duration, sec
21
+      c1: Throughput, msg/sec
22
+      d1: Transfer, Mb
23
+      d2: Bandwidth, Mb/sec
24
+      e1: Avg. latency, ms
25
+      e2: Max latency, ms
21 26
     collection: records
22 27
     pipeline:
23 28
     - $match: { task: omsimulator, mode: notify }
24 29
     - $project:
25 30
         a1: "$client.count"
26
-        b1: "$server.count"
27
-        b2: { $subtract: ["$client.count", "$server.count" ] }
28
-        c1: { $multiply: ["$server.latency", 1000] }
29
-        c2: { $multiply: ["$server.max_latency", 1000] }
31
+        a2: "$server.count"
32
+        a3: { $subtract: ["$client.count", "$server.count" ] }
33
+        a4: "$error.count"
34
+        b1: "$client.duration"
35
+        c1: { $divide: ["$client.count", "$client.duration"] }
36
+        d1: { $divide: ["$client.size", 1048576] }
37
+        d2: { $divide: [$divide: ["$client.size", 1048576], "$client.duration"] }
38
+        e1: { $multiply: ["$server.latency", 1000] }
39
+        e2: { $multiply: ["$server.max_latency", 1000] }
30 40
 ''' | info
31 41
 }}
32 42
 
33
-**Message flow**
43
+Message flow
44
+^^^^^^^^^^^^
45
+
46
+This chart shows the message flow between client and server. It includes
47
+messages sent by the client, received by the server and errors caught by
48
+the client.
34 49
 
35 50
 {{'''
36 51
     title: NOTIFY message flow
@@ -38,7 +53,7 @@ NOTIFY fail-over throughput test
38 53
       x: time
39 54
       y1: sent, msg
40 55
       y2: received, msg
41
-      y3: latency, ms
56
+      y3: errors, msg
42 57
     chart: line
43 58
     collection: series
44 59
     pipelines:
@@ -52,29 +67,51 @@ NOTIFY fail-over throughput test
52 67
       - $project:
53 68
           x: "$timestamp"
54 69
           y2: "$count"
55
-          y3: { $multiply: ["$latency", 1000] }
70
+    -
71
+      - $match: { task: omsimulator, mode: notify, name: error_0 }
72
+      - $project:
73
+          x: "$timestamp"
74
+          y3: "$count"
56 75
 ''' | chart
57 76
 }}
58 77
 
78
+where:
79
+ * ``sent`` - messages sent by the client
80
+ * ``received`` - messages received by the server
81
+ * ``errors`` - errors exposed and caught by the client
59 82
 
60
-**Messages sent by the client**
83
+
84
+Messages sent by the client
85
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
86
+
87
+This chart shows messages sent by client and error rate.
61 88
 
62 89
 {{'''
63 90
     title: NOTIFY sent messages
64 91
     axes:
65 92
       x: time
66 93
       y: sent, msg
94
+      y2: errors, msg
67 95
     chart: line
68 96
     collection: series
69
-    pipeline:
70
-    - $match: { task: omsimulator, mode: notify, name: client_0 }
71
-    - $project:
72
-        x: "$seq"
73
-        y: "$count"
97
+    pipelines:
98
+    -
99
+      - $match: { task: omsimulator, mode: notify, name: client_0 }
100
+      - $project:
101
+          x: "$seq"
102
+          y: "$count"
103
+    -
104
+      - $match: { task: omsimulator, mode: notify, name: error_0 }
105
+      - $project:
106
+          x: "$seq"
107
+          y2: "$count"
74 108
 ''' | chart_and_table
75 109
 }}
76 110
 
77
-**Messages received by the server**
111
+Messages received by the server
112
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
113
+
114
+This chart shows messages received by the server and their latency.
78 115
 
79 116
 {{'''
80 117
     title: NOTIFY received messages

Loading…
Cancel
Save