Remove memory stats and optimize match operator
This commit is contained in:
parent
526a1ea35d
commit
fbc1679790
|
@ -52,11 +52,9 @@ def aggregate(scenario, mongo_url, db_name, tag):
|
||||||
stop = rec['end'] # todo rename field into 'stop'
|
stop = rec['end'] # todo rename field into 'stop'
|
||||||
|
|
||||||
series_pipeline = [
|
series_pipeline = [
|
||||||
{'$match': {'$and': [
|
{'$match': {
|
||||||
{'tag': tag},
|
'tag': tag,
|
||||||
{'timestamp': {'$gte': start}},
|
'timestamp': {'$gte': start, '$lte': stop}}}
|
||||||
{'timestamp': {'$lte': stop}}
|
|
||||||
]}}
|
|
||||||
]
|
]
|
||||||
series_pipeline.extend(values_pipeline)
|
series_pipeline.extend(values_pipeline)
|
||||||
|
|
||||||
|
|
|
@ -84,7 +84,7 @@ execution:
|
||||||
tasks:
|
tasks:
|
||||||
- atop:
|
- atop:
|
||||||
command: stop
|
command: stop
|
||||||
labels: [ PRC, PRM ]
|
labels: [ PRC ]
|
||||||
|
|
||||||
aggregation:
|
aggregation:
|
||||||
-
|
-
|
||||||
|
@ -95,14 +95,6 @@ aggregation:
|
||||||
pipeline:
|
pipeline:
|
||||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
||||||
- $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
|
- $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||||
-
|
|
||||||
update:
|
|
||||||
query:
|
|
||||||
{ task: omsimulator }
|
|
||||||
values:
|
|
||||||
pipeline:
|
|
||||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
|
||||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
|
||||||
|
|
||||||
{% if rabbit_hosts[1] %}
|
{% if rabbit_hosts[1] %}
|
||||||
-
|
-
|
||||||
|
@ -113,14 +105,6 @@ aggregation:
|
||||||
pipeline:
|
pipeline:
|
||||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
||||||
- $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
|
- $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||||
-
|
|
||||||
update:
|
|
||||||
query:
|
|
||||||
{ task: omsimulator }
|
|
||||||
values:
|
|
||||||
pipeline:
|
|
||||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
|
||||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
{% if rabbit_hosts[2] %}
|
{% if rabbit_hosts[2] %}
|
||||||
|
@ -132,14 +116,6 @@ aggregation:
|
||||||
pipeline:
|
pipeline:
|
||||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
||||||
- $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
|
- $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||||
-
|
|
||||||
update:
|
|
||||||
query:
|
|
||||||
{ task: omsimulator }
|
|
||||||
values:
|
|
||||||
pipeline:
|
|
||||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
|
||||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
report:
|
report:
|
||||||
|
|
|
@ -55,28 +55,22 @@ depending on number of concurrent threads.
|
||||||
title: RPC CALL throughput, latency and RabbitMQ CPU utilization depending on thread count
|
title: RPC CALL throughput, latency and RabbitMQ CPU utilization depending on thread count
|
||||||
axes:
|
axes:
|
||||||
x: threads
|
x: threads
|
||||||
y: sent, msg/sec
|
y3: throughput, msg/sec
|
||||||
y2: received, msg/sec
|
|
||||||
y3: round-trip, msg/sec
|
|
||||||
y4: latency, ms
|
y4: latency, ms
|
||||||
y5: RabbitMQ CPU consumption, %
|
y5: RabbitMQ CPU, %
|
||||||
chart: line
|
chart: line
|
||||||
pipeline:
|
pipeline:
|
||||||
- $match: { task: omsimulator, mode: call }
|
- $match: { task: omsimulator, mode: call }
|
||||||
- $group:
|
- $group:
|
||||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||||
msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
|
throughput: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }}
|
||||||
msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
|
|
||||||
msg_round_trip_per_sec: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }}
|
|
||||||
latency: { $avg: "$round_trip.latency" }
|
latency: { $avg: "$round_trip.latency" }
|
||||||
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
||||||
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
||||||
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
||||||
- $project:
|
- $project:
|
||||||
x: "$_id.threads"
|
x: "$_id.threads"
|
||||||
y: "$msg_sent_per_sec"
|
y3: "$throughput"
|
||||||
y2: "$msg_received_per_sec"
|
|
||||||
y3: "$msg_round_trip_per_sec"
|
|
||||||
y4: { $multiply: [ "$latency", 1000 ] }
|
y4: { $multiply: [ "$latency", 1000 ] }
|
||||||
y5: { $multiply: [ { $add: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
|
y5: { $multiply: [ { $add: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
|
||||||
''' | chart_and_table
|
''' | chart_and_table
|
||||||
|
@ -120,33 +114,6 @@ Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
|
||||||
''' | chart_and_table
|
''' | chart_and_table
|
||||||
}}
|
}}
|
||||||
|
|
||||||
Detailed RabbitMQ resident memory
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
|
|
||||||
|
|
||||||
{{'''
|
|
||||||
title: RabbitMQ nodes memory consumption during RPC CALL load test
|
|
||||||
axes:
|
|
||||||
x: threads
|
|
||||||
y0: Master, Mb
|
|
||||||
y1: Slave 1, Mb
|
|
||||||
y2: Slave 2, Mb
|
|
||||||
chart: line
|
|
||||||
pipeline:
|
|
||||||
- $match: { task: omsimulator, mode: call }
|
|
||||||
- $group:
|
|
||||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
|
||||||
rabbit_0: { $avg: "$rabbit_resident_0" }
|
|
||||||
rabbit_1: { $avg: "$rabbit_resident_1" }
|
|
||||||
rabbit_2: { $avg: "$rabbit_resident_2" }
|
|
||||||
- $project:
|
|
||||||
x: "$_id.threads"
|
|
||||||
y0: { $divide: [ "$rabbit_0", 1048576 ] }
|
|
||||||
y1: { $divide: [ "$rabbit_1", 1048576 ] }
|
|
||||||
y2: { $divide: [ "$rabbit_2", 1048576 ] }
|
|
||||||
''' | chart_and_table
|
|
||||||
}}
|
|
||||||
|
|
||||||
|
|
||||||
Test Case 2: RPC CAST Throughput Test
|
Test Case 2: RPC CAST Throughput Test
|
||||||
|
@ -195,8 +162,7 @@ depending on number of concurrent threads.
|
||||||
title: RPC CAST throughput, latency and RabbitMQ CPU utilization depending on thread count
|
title: RPC CAST throughput, latency and RabbitMQ CPU utilization depending on thread count
|
||||||
axes:
|
axes:
|
||||||
x: threads
|
x: threads
|
||||||
y: sent, msg/sec
|
y: throughput, msg/sec
|
||||||
y2: received, msg/sec
|
|
||||||
y3: latency, ms
|
y3: latency, ms
|
||||||
y4: RabbitMQ CPU consumption, %
|
y4: RabbitMQ CPU consumption, %
|
||||||
chart: line
|
chart: line
|
||||||
|
@ -204,16 +170,14 @@ depending on number of concurrent threads.
|
||||||
- $match: { task: omsimulator, mode: cast }
|
- $match: { task: omsimulator, mode: cast }
|
||||||
- $group:
|
- $group:
|
||||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||||
msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
|
throughput: { $sum: { $divide: ["$client.count", "$client.duration"] }}
|
||||||
msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
|
|
||||||
latency: { $avg: "$server.latency" }
|
latency: { $avg: "$server.latency" }
|
||||||
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
||||||
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
||||||
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
||||||
- $project:
|
- $project:
|
||||||
x: "$_id.threads"
|
x: "$_id.threads"
|
||||||
y: "$msg_sent_per_sec"
|
y: "$throughput"
|
||||||
y2: "$msg_received_per_sec"
|
|
||||||
y3: { $multiply: [ "$latency", 1000 ] }
|
y3: { $multiply: [ "$latency", 1000 ] }
|
||||||
y4: { $multiply: [ { $add: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
|
y4: { $multiply: [ { $add: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
|
||||||
''' | chart_and_table
|
''' | chart_and_table
|
||||||
|
@ -256,34 +220,6 @@ Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
|
||||||
''' | chart_and_table
|
''' | chart_and_table
|
||||||
}}
|
}}
|
||||||
|
|
||||||
Detailed RabbitMQ resident memory
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
|
|
||||||
|
|
||||||
{{'''
|
|
||||||
title: RabbitMQ nodes memory consumption during RPC CAST load test
|
|
||||||
axes:
|
|
||||||
x: threads
|
|
||||||
y0: Master, Mb
|
|
||||||
y1: Slave 1, Mb
|
|
||||||
y2: Slave 2, Mb
|
|
||||||
chart: line
|
|
||||||
pipeline:
|
|
||||||
- $match: { task: omsimulator, mode: cast }
|
|
||||||
- $group:
|
|
||||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
|
||||||
rabbit_0: { $avg: "$rabbit_resident_0" }
|
|
||||||
rabbit_1: { $avg: "$rabbit_resident_1" }
|
|
||||||
rabbit_2: { $avg: "$rabbit_resident_2" }
|
|
||||||
- $project:
|
|
||||||
x: "$_id.threads"
|
|
||||||
y0: { $divide: [ "$rabbit_0", 1048576 ] }
|
|
||||||
y1: { $divide: [ "$rabbit_1", 1048576 ] }
|
|
||||||
y2: { $divide: [ "$rabbit_2", 1048576 ] }
|
|
||||||
''' | chart_and_table
|
|
||||||
}}
|
|
||||||
|
|
||||||
|
|
||||||
Test Case 3: Notification Throughput Test
|
Test Case 3: Notification Throughput Test
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
@ -331,8 +267,7 @@ depending on number of concurrent threads.
|
||||||
title: NOTIFY throughput, latency and RabbitMQ CPU utilization depending on thread count
|
title: NOTIFY throughput, latency and RabbitMQ CPU utilization depending on thread count
|
||||||
axes:
|
axes:
|
||||||
x: threads
|
x: threads
|
||||||
y: sent, msg/sec
|
y: throughput, msg/sec
|
||||||
y2: received, msg/sec
|
|
||||||
y3: latency, ms
|
y3: latency, ms
|
||||||
y4: RabbitMQ CPU consumption, %
|
y4: RabbitMQ CPU consumption, %
|
||||||
chart: line
|
chart: line
|
||||||
|
@ -340,16 +275,14 @@ depending on number of concurrent threads.
|
||||||
- $match: { task: omsimulator, mode: notify }
|
- $match: { task: omsimulator, mode: notify }
|
||||||
- $group:
|
- $group:
|
||||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
||||||
msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
|
throughput: { $sum: { $divide: ["$client.count", "$client.duration"] }}
|
||||||
msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
|
|
||||||
latency: { $avg: "$server.latency" }
|
latency: { $avg: "$server.latency" }
|
||||||
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
rabbit_total_0: { $avg: "$rabbit_total_0" }
|
||||||
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
rabbit_total_1: { $avg: "$rabbit_total_1" }
|
||||||
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
rabbit_total_2: { $avg: "$rabbit_total_2" }
|
||||||
- $project:
|
- $project:
|
||||||
x: "$_id.threads"
|
x: "$_id.threads"
|
||||||
y: "$msg_sent_per_sec"
|
y: "$throughput"
|
||||||
y2: "$msg_received_per_sec"
|
|
||||||
y3: { $multiply: [ "$latency", 1000 ] }
|
y3: { $multiply: [ "$latency", 1000 ] }
|
||||||
y4: { $multiply: [ { $add: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
|
y4: { $multiply: [ { $add: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
|
||||||
''' | chart_and_table
|
''' | chart_and_table
|
||||||
|
@ -392,34 +325,6 @@ Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
|
||||||
''' | chart_and_table
|
''' | chart_and_table
|
||||||
}}
|
}}
|
||||||
|
|
||||||
Detailed RabbitMQ resident memory
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
|
|
||||||
|
|
||||||
{{'''
|
|
||||||
title: RabbitMQ nodes memory consumption during NOTIFY load test
|
|
||||||
axes:
|
|
||||||
x: threads
|
|
||||||
y0: Master, Mb
|
|
||||||
y1: Slave 1, Mb
|
|
||||||
y2: Slave 2, Mb
|
|
||||||
chart: line
|
|
||||||
pipeline:
|
|
||||||
- $match: { task: omsimulator, mode: notify }
|
|
||||||
- $group:
|
|
||||||
_id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
|
|
||||||
rabbit_0: { $avg: "$rabbit_resident_0" }
|
|
||||||
rabbit_1: { $avg: "$rabbit_resident_1" }
|
|
||||||
rabbit_2: { $avg: "$rabbit_resident_2" }
|
|
||||||
- $project:
|
|
||||||
x: "$_id.threads"
|
|
||||||
y0: { $divide: [ "$rabbit_0", 1048576 ] }
|
|
||||||
y1: { $divide: [ "$rabbit_1", 1048576 ] }
|
|
||||||
y2: { $divide: [ "$rabbit_2", 1048576 ] }
|
|
||||||
''' | chart_and_table
|
|
||||||
}}
|
|
||||||
|
|
||||||
|
|
||||||
.. references:
|
.. references:
|
||||||
|
|
||||||
|
|
|
@ -87,7 +87,7 @@ execution:
|
||||||
tasks:
|
tasks:
|
||||||
- atop:
|
- atop:
|
||||||
command: stop
|
command: stop
|
||||||
labels: [ PRC, PRM ]
|
labels: [ PRC ]
|
||||||
|
|
||||||
aggregation:
|
aggregation:
|
||||||
-
|
-
|
||||||
|
@ -98,14 +98,6 @@ aggregation:
|
||||||
pipeline:
|
pipeline:
|
||||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
||||||
- $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
|
- $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||||
-
|
|
||||||
update:
|
|
||||||
query:
|
|
||||||
{ task: omsimulator }
|
|
||||||
values:
|
|
||||||
pipeline:
|
|
||||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
|
|
||||||
- $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
|
|
||||||
|
|
||||||
{% if rabbit_hosts[1] %}
|
{% if rabbit_hosts[1] %}
|
||||||
-
|
-
|
||||||
|
@ -116,14 +108,6 @@ aggregation:
|
||||||
pipeline:
|
pipeline:
|
||||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
||||||
- $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
|
- $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||||
-
|
|
||||||
update:
|
|
||||||
query:
|
|
||||||
{ task: omsimulator }
|
|
||||||
values:
|
|
||||||
pipeline:
|
|
||||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
|
|
||||||
- $group: { _id: null, rabbit_resident_1: { $avg: "$resident" } }
|
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
{% if rabbit_hosts[2] %}
|
{% if rabbit_hosts[2] %}
|
||||||
|
@ -135,14 +119,6 @@ aggregation:
|
||||||
pipeline:
|
pipeline:
|
||||||
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
- $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
||||||
- $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
|
- $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
|
||||||
-
|
|
||||||
update:
|
|
||||||
query:
|
|
||||||
{ task: omsimulator }
|
|
||||||
values:
|
|
||||||
pipeline:
|
|
||||||
- $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
|
|
||||||
- $group: { _id: null, rabbit_resident_2: { $avg: "$resident" } }
|
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
report:
|
report:
|
||||||
|
|
Loading…
Reference in New Issue