Browse Source

Remove memory stats and optimize match operator

Ilya Shakhat 3 years ago
parent
commit
fbc1679790

+ 3
- 5
performa/engine/aggregator.py View File

@@ -52,11 +52,9 @@ def aggregate(scenario, mongo_url, db_name, tag):
52 52
                     stop = rec['end']  # todo rename field into 'stop'
53 53
 
54 54
                     series_pipeline = [
55
-                        {'$match': {'$and': [
56
-                            {'tag': tag},
57
-                            {'timestamp': {'$gte': start}},
58
-                            {'timestamp': {'$lte': stop}}
59
-                        ]}}
55
+                        {'$match': {
56
+                            'tag': tag,
57
+                            'timestamp': {'$gte': start, '$lte': stop}}}
60 58
                     ]
61 59
                     series_pipeline.extend(values_pipeline)
62 60
 

+ 1
- 25
performa/scenarios/mq/omsimulator-threading.yaml View File

@@ -84,7 +84,7 @@ execution:
84 84
     tasks:
85 85
     - atop:
86 86
         command: stop
87
-        labels: [ PRC, PRM ]
87
+        labels: [ PRC ]
88 88
 
89 89
 aggregation:
90 90
   -
@@ -95,14 +95,6 @@ aggregation:
95 95
         pipeline:
96 96
         - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
97 97
         - $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
98
-  -
99
-    update:
100
-      query:
101
-        { task: omsimulator }
102
-      values:
103
-        pipeline:
104
-        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
105
-        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
106 98
 
107 99
 {% if rabbit_hosts[1] %}
108 100
   -
@@ -113,14 +105,6 @@ aggregation:
113 105
         pipeline:
114 106
         - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
115 107
         - $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
116
-  -
117
-    update:
118
-      query:
119
-        { task: omsimulator }
120
-      values:
121
-        pipeline:
122
-        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
123
-        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
124 108
 {% endif %}
125 109
 
126 110
 {% if rabbit_hosts[2] %}
@@ -132,14 +116,6 @@ aggregation:
132 116
         pipeline:
133 117
         - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
134 118
         - $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
135
-  -
136
-    update:
137
-      query:
138
-        { task: omsimulator }
139
-      values:
140
-        pipeline:
141
-        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
142
-        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
143 119
 {% endif %}
144 120
 
145 121
 report:

+ 10
- 105
performa/scenarios/mq/omsimulator.rst View File

@@ -55,28 +55,22 @@ depending on number of concurrent threads.
55 55
     title: RPC CALL throughput, latency and RabbitMQ CPU utilization depending on thread count
56 56
     axes:
57 57
       x: threads
58
-      y: sent, msg/sec
59
-      y2: received, msg/sec
60
-      y3: round-trip, msg/sec
58
+      y3: throughput, msg/sec
61 59
       y4: latency, ms
62
-      y5: RabbitMQ CPU consumption, %
60
+      y5: RabbitMQ CPU, %
63 61
     chart: line
64 62
     pipeline:
65 63
     - $match: { task: omsimulator, mode: call }
66 64
     - $group:
67 65
         _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
68
-        msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
69
-        msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
70
-        msg_round_trip_per_sec: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }}
66
+        throughput: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }}
71 67
         latency: { $avg: "$round_trip.latency" }
72 68
         rabbit_total_0: { $avg: "$rabbit_total_0" }
73 69
         rabbit_total_1: { $avg: "$rabbit_total_1" }
74 70
         rabbit_total_2: { $avg: "$rabbit_total_2" }
75 71
     - $project:
76 72
         x: "$_id.threads"
77
-        y: "$msg_sent_per_sec"
78
-        y2: "$msg_received_per_sec"
79
-        y3: "$msg_round_trip_per_sec"
73
+        y3: "$throughput"
80 74
         y4: { $multiply: [ "$latency", 1000 ] }
81 75
         y5: { $multiply: [ { $add: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
82 76
 ''' | chart_and_table
@@ -120,33 +114,6 @@ Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
120 114
 ''' | chart_and_table
121 115
 }}
122 116
 
123
-Detailed RabbitMQ resident memory
124
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
125
-
126
-Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
127
-
128
-{{'''
129
-    title: RabbitMQ nodes memory consumption during RPC CALL load test
130
-    axes:
131
-      x: threads
132
-      y0: Master, Mb
133
-      y1: Slave 1, Mb
134
-      y2: Slave 2, Mb
135
-    chart: line
136
-    pipeline:
137
-    - $match: { task: omsimulator, mode: call }
138
-    - $group:
139
-        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
140
-        rabbit_0: { $avg: "$rabbit_resident_0" }
141
-        rabbit_1: { $avg: "$rabbit_resident_1" }
142
-        rabbit_2: { $avg: "$rabbit_resident_2" }
143
-    - $project:
144
-        x: "$_id.threads"
145
-        y0: { $divide: [ "$rabbit_0", 1048576 ] }
146
-        y1: { $divide: [ "$rabbit_1", 1048576 ] }
147
-        y2: { $divide: [ "$rabbit_2", 1048576 ] }
148
-''' | chart_and_table
149
-}}
150 117
 
151 118
 
152 119
 Test Case 2: RPC CAST Throughput Test
@@ -195,8 +162,7 @@ depending on number of concurrent threads.
195 162
     title: RPC CAST throughput, latency and RabbitMQ CPU utilization depending on thread count
196 163
     axes:
197 164
       x: threads
198
-      y: sent, msg/sec
199
-      y2: received, msg/sec
165
+      y: throughput, msg/sec
200 166
       y3: latency, ms
201 167
       y4: RabbitMQ CPU consumption, %
202 168
     chart: line
@@ -204,16 +170,14 @@ depending on number of concurrent threads.
204 170
     - $match: { task: omsimulator, mode: cast }
205 171
     - $group:
206 172
         _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
207
-        msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
208
-        msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
173
+        throughput: { $sum: { $divide: ["$client.count", "$client.duration"] }}
209 174
         latency: { $avg: "$server.latency" }
210 175
         rabbit_total_0: { $avg: "$rabbit_total_0" }
211 176
         rabbit_total_1: { $avg: "$rabbit_total_1" }
212 177
         rabbit_total_2: { $avg: "$rabbit_total_2" }
213 178
     - $project:
214 179
         x: "$_id.threads"
215
-        y: "$msg_sent_per_sec"
216
-        y2: "$msg_received_per_sec"
180
+        y: "$throughput"
217 181
         y3: { $multiply: [ "$latency", 1000 ] }
218 182
         y4: { $multiply: [ { $add: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
219 183
 ''' | chart_and_table
@@ -256,34 +220,6 @@ Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
256 220
 ''' | chart_and_table
257 221
 }}
258 222
 
259
-Detailed RabbitMQ resident memory
260
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
261
-
262
-Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
263
-
264
-{{'''
265
-    title: RabbitMQ nodes memory consumption during RPC CAST load test
266
-    axes:
267
-      x: threads
268
-      y0: Master, Mb
269
-      y1: Slave 1, Mb
270
-      y2: Slave 2, Mb
271
-    chart: line
272
-    pipeline:
273
-    - $match: { task: omsimulator, mode: cast }
274
-    - $group:
275
-        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
276
-        rabbit_0: { $avg: "$rabbit_resident_0" }
277
-        rabbit_1: { $avg: "$rabbit_resident_1" }
278
-        rabbit_2: { $avg: "$rabbit_resident_2" }
279
-    - $project:
280
-        x: "$_id.threads"
281
-        y0: { $divide: [ "$rabbit_0", 1048576 ] }
282
-        y1: { $divide: [ "$rabbit_1", 1048576 ] }
283
-        y2: { $divide: [ "$rabbit_2", 1048576 ] }
284
-''' | chart_and_table
285
-}}
286
-
287 223
 
288 224
 Test Case 3: Notification Throughput Test
289 225
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -331,8 +267,7 @@ depending on number of concurrent threads.
331 267
     title: NOTIFY throughput, latency and RabbitMQ CPU utilization depending on thread count
332 268
     axes:
333 269
       x: threads
334
-      y: sent, msg/sec
335
-      y2: received, msg/sec
270
+      y: throughput, msg/sec
336 271
       y3: latency, ms
337 272
       y4: RabbitMQ CPU consumption, %
338 273
     chart: line
@@ -340,16 +275,14 @@ depending on number of concurrent threads.
340 275
     - $match: { task: omsimulator, mode: notify }
341 276
     - $group:
342 277
         _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
343
-        msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
344
-        msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
278
+        throughput: { $sum: { $divide: ["$client.count", "$client.duration"] }}
345 279
         latency: { $avg: "$server.latency" }
346 280
         rabbit_total_0: { $avg: "$rabbit_total_0" }
347 281
         rabbit_total_1: { $avg: "$rabbit_total_1" }
348 282
         rabbit_total_2: { $avg: "$rabbit_total_2" }
349 283
     - $project:
350 284
         x: "$_id.threads"
351
-        y: "$msg_sent_per_sec"
352
-        y2: "$msg_received_per_sec"
285
+        y: "$throughput"
353 286
         y3: { $multiply: [ "$latency", 1000 ] }
354 287
         y4: { $multiply: [ { $add: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
355 288
 ''' | chart_and_table
@@ -392,34 +325,6 @@ Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
392 325
 ''' | chart_and_table
393 326
 }}
394 327
 
395
-Detailed RabbitMQ resident memory
396
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
397
-
398
-Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
399
-
400
-{{'''
401
-    title: RabbitMQ nodes memory consumption during NOTIFY load test
402
-    axes:
403
-      x: threads
404
-      y0: Master, Mb
405
-      y1: Slave 1, Mb
406
-      y2: Slave 2, Mb
407
-    chart: line
408
-    pipeline:
409
-    - $match: { task: omsimulator, mode: notify }
410
-    - $group:
411
-        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
412
-        rabbit_0: { $avg: "$rabbit_resident_0" }
413
-        rabbit_1: { $avg: "$rabbit_resident_1" }
414
-        rabbit_2: { $avg: "$rabbit_resident_2" }
415
-    - $project:
416
-        x: "$_id.threads"
417
-        y0: { $divide: [ "$rabbit_0", 1048576 ] }
418
-        y1: { $divide: [ "$rabbit_1", 1048576 ] }
419
-        y2: { $divide: [ "$rabbit_2", 1048576 ] }
420
-''' | chart_and_table
421
-}}
422
-
423 328
 
424 329
 .. references:
425 330
 

+ 1
- 25
performa/scenarios/mq/omsimulator.yaml View File

@@ -87,7 +87,7 @@ execution:
87 87
     tasks:
88 88
     - atop:
89 89
         command: stop
90
-        labels: [ PRC, PRM ]
90
+        labels: [ PRC ]
91 91
 
92 92
 aggregation:
93 93
   -
@@ -98,14 +98,6 @@ aggregation:
98 98
         pipeline:
99 99
         - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
100 100
         - $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
101
-  -
102
-    update:
103
-      query:
104
-        { task: omsimulator }
105
-      values:
106
-        pipeline:
107
-        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
108
-        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
109 101
 
110 102
 {% if rabbit_hosts[1] %}
111 103
   -
@@ -116,14 +108,6 @@ aggregation:
116 108
         pipeline:
117 109
         - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
118 110
         - $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
119
-  -
120
-    update:
121
-      query:
122
-        { task: omsimulator }
123
-      values:
124
-        pipeline:
125
-        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
126
-        - $group: { _id: null, rabbit_resident_1: { $avg: "$resident" } }
127 111
 {% endif %}
128 112
 
129 113
 {% if rabbit_hosts[2] %}
@@ -135,14 +119,6 @@ aggregation:
135 119
         pipeline:
136 120
         - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
137 121
         - $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
138
-  -
139
-    update:
140
-      query:
141
-        { task: omsimulator }
142
-      values:
143
-        pipeline:
144
-        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
145
-        - $group: { _id: null, rabbit_resident_2: { $avg: "$resident" } }
146 122
 {% endif %}
147 123
 
148 124
 report:

Loading…
Cancel
Save