Browse Source

Run multiple omsimulator pairs on a single node

Ilya Shakhat 3 years ago
parent
commit
67c1957349

+ 102
- 95
performa/modules/omsimulator.py View File

@@ -1,5 +1,6 @@
1 1
 #!/usr/bin/python
2 2
 import copy
3
+import multiprocessing
3 4
 import os
4 5
 import signal
5 6
 import tempfile
@@ -11,35 +12,6 @@ UNIQUE_NAME = 'performa_omsimulator'
11 12
 DIR = '/tmp/performa/oslo.messaging/tools/'
12 13
 PYTHON = '/tmp/performa/oslo.messaging/.venv/bin/python'
13 14
 
14
-PATTERNS = [
15
-    r'(?P<msg_sent>\d+) messages were sent for (?P<duration>\d+) sec',
16
-    r'(?P<bytes_sent>\d+) bytes were sent for',
17
-]
18
-TRANSFORM_FIELDS = {
19
-    'msg_sent': int,
20
-    'bytes_sent': int,
21
-    'duration': int,
22
-}
23
-
24
-
25
-def parse_output(raw):
26
-    result = {}
27
-
28
-    for pattern in PATTERNS:
29
-        for parsed in re.finditer(pattern, raw):
30
-            result.update(parsed.groupdict())
31
-
32
-    for k in result.keys():
33
-        if k in TRANSFORM_FIELDS:
34
-            result[k] = TRANSFORM_FIELDS[k](result[k])
35
-
36
-    result['msg_sent_bandwidth'] = (result.get('msg_sent', 0) /
37
-                                    result.get('duration', 1))
38
-    result['bytes_sent_bandwidth'] = (result.get('bytes_sent', 0) /
39
-                                      result.get('duration', 1))
40
-
41
-    return result
42
-
43 15
 
44 16
 def chdir(module):
45 17
     try:
@@ -48,29 +20,6 @@ def chdir(module):
48 20
         module.fail_json(msg='Failed to change dir to %s: %s' % (DIR, e))
49 21
 
50 22
 
51
-def start_daemon(module, cmd):
52
-    cmd = ('daemon -n %(name)s -D %(dir)s -F %(pid)s -U -- %(cmd)s' %
53
-           dict(name=UNIQUE_NAME, dir=DIR, pid=SERVER_PID, cmd=cmd))
54
-
55
-    rc, stdout, stderr = module.run_command(cmd)
56
-    result = dict(changed=True, rc=rc, stdout=stdout, stderr=stderr, cmd=cmd)
57
-
58
-    if rc:
59
-        module.fail_json(msg='Failed to start omsimulator', **result)
60
-
61
-
62
-def stop_daemon(module):
63
-    rc, stdout, stderr = module.run_command('/bin/cat %s' % SERVER_PID)
64
-
65
-    if rc:
66
-        return
67
-
68
-    rc, stdout, stderr = module.run_command('pgrep -P %s' % stdout)
69
-    os.kill(int(stdout), signal.SIGINT)
70
-
71
-    time.sleep(2)
72
-
73
-
74 23
 def read_file(filename):
75 24
     fd = None
76 25
     try:
@@ -92,35 +41,16 @@ def transform_series(series):
92 41
     return result
93 42
 
94 43
 
95
-def run(module):
96
-    params = copy.deepcopy(module.params)
44
+def make_file_name(base, index):
45
+    return '%s-%s' % (base, index)
97 46
 
98
-    if params['mode'] == 'notify':
99
-        server_tool = 'notify-server'
100
-        client_tool = 'notify-client'
101
-    else:
102
-        server_tool = 'rpc-server'
103
-        client_tool = 'rpc-client'
104
-
105
-    params['python'] = PYTHON
106
-    # todo: fix topic support in omsimulator
107
-    # params['topic'] = 'performa-%d' % (random.random() * 1000000)
108
-    params['server_tool'] = server_tool
109
-    params['client_tool'] = client_tool
110
-    params['server_file'] = SERVER_FILE_NAME
111
-    params['client_file'] = CLIENT_FILE_NAME
112
-
113
-    params['url'] = params['server_url'] or params['url']
114
-    server = ('%(python)s simulator.py '
115
-              # '--topic %(topic)s '
116
-              '--url %(url)s '
117
-              '--json %(server_file)s '
118
-              '%(server_tool)s ') % params
119 47
 
48
+def make_client_cmd(params, i):
49
+    params['client_file'] = make_file_name(CLIENT_FILE_NAME, i)
120 50
     params['url'] = params['client_url'] or params['url']
51
+
121 52
     client = ('%(python)s simulator.py '
122
-              # '--topic %(topic)s '
123
-              '--url=%(url)s '
53
+              '--url %(url)s '
124 54
               '--json %(client_file)s '
125 55
               '-l %(duration)s '
126 56
               '%(client_tool)s '
@@ -134,45 +64,121 @@ def run(module):
134 64
     if params['mode'] == 'fanout':
135 65
         client += '--is-fanout True '
136 66
 
137
-    start_daemon(module, server)
67
+    return client
138 68
 
139
-    rc, stdout, stderr = module.run_command(client)
140 69
 
141
-    if rc:
142
-        module.fail_json(msg='Failed to start omsimulator',
143
-                         stderr=stderr, rc=rc, cmd=client)
70
+def make_server_cmd(params, i):
71
+    params['server_file'] = make_file_name(SERVER_FILE_NAME, i)
72
+    params['url'] = params['server_url'] or params['url']
144 73
 
145
-    stop_daemon(module)
74
+    server = ('%(python)s simulator.py '
75
+              '--url %(url)s '
76
+              '--json %(server_file)s '
77
+              '%(server_tool)s ') % params
146 78
 
147
-    try:
148
-        client_data = read_file(CLIENT_FILE_NAME)
149
-        server_data = read_file(SERVER_FILE_NAME)
79
+    return server
150 80
 
151
-        client_summary = client_data['summary']['client']
152 81
 
153
-        record = dict(start=client_summary['start'],
154
-                      end=client_summary['end'],
155
-                      client=client_summary)
82
+def run_client(module, command):
83
+    module.run_command(command)
156 84
 
85
+
86
+def run_client_pool(module, params):
87
+    processes = []
88
+
89
+    for i in range(params['processes']):
90
+        cmd = make_client_cmd(params, i)
91
+        p = multiprocessing.Process(target=run_client, args=(module, cmd))
92
+        processes.append(p)
93
+        p.start()
94
+
95
+    for p in processes:
96
+        p.join()
97
+
98
+
99
+def run_server(module, command):
100
+    module.run_command(command)
101
+
102
+
103
+def start_server_pool(module, params):
104
+    processes = []
105
+
106
+    for i in range(params['processes']):
107
+        cmd = make_server_cmd(params, i)
108
+        p = multiprocessing.Process(target=run_client, args=(module, cmd))
109
+        processes.append(p)
110
+        p.start()
111
+
112
+    return processes
113
+
114
+
115
+def stop_server_pool(module, processes):
116
+    for p in processes:
117
+        rc, stdout, stderr = module.run_command('pgrep -P %s' % p.pid)
118
+
119
+        for child in (int(p) for p in stdout.split('\n') if p):
120
+            os.kill(child, signal.SIGINT)
121
+
122
+    time.sleep(3)  # let simulators handle the signal
123
+
124
+    for p in processes:
125
+        os.kill(p.pid, signal.SIGINT)
126
+        p.join()
127
+
128
+
129
+def collect_results(params):
130
+    result = dict(records=[], series=[])
131
+
132
+    for i in range(params['processes']):
133
+        client_data = read_file(make_file_name(CLIENT_FILE_NAME, i))
134
+        server_data = read_file(make_file_name(SERVER_FILE_NAME, i))
135
+
136
+        client_summary = client_data['summary']['client']
137
+        record = dict(start=client_summary['start'], end=client_summary['end'],
138
+                      client=client_summary)
157 139
         if 'round_trip' in client_data['summary']:
158 140
             round_trip_summary = client_data['summary']['round_trip']
159 141
             record['round_trip'] = round_trip_summary
160
-
161 142
         if 'error' in client_data['summary']:
162 143
             error_summary = client_data['summary']['error']
163 144
             record['error'] = error_summary
164
-
165 145
         server_summary = server_data['summary']
166 146
         record['server'] = server_summary
167
-
168 147
         series = transform_series(client_data['series'])
169 148
         series.extend(transform_series(server_data['series']))
170 149
 
171
-        result = dict(records=[record], series=series)
150
+        result['records'].append(record)
151
+        result['series'] += series
152
+
153
+    return result
154
+
155
+
156
+def run(module):
157
+    params = copy.deepcopy(module.params)
158
+
159
+    if params['mode'] == 'notify':
160
+        server_tool = 'notify-server'
161
+        client_tool = 'notify-client'
162
+    else:
163
+        server_tool = 'rpc-server'
164
+        client_tool = 'rpc-client'
165
+
166
+    params['python'] = PYTHON
167
+    params['server_tool'] = server_tool
168
+    params['client_tool'] = client_tool
169
+
170
+    server_processes = start_server_pool(module, params)
171
+
172
+    run_client_pool(module, params)
173
+
174
+    stop_server_pool(module, server_processes)
175
+
176
+    try:
177
+        result = collect_results(params)
172 178
         module.exit_json(**result)
173 179
     except Exception as e:
174 180
         msg = 'Failed to read omsimulator output: %s' % e
175
-        module.fail_json(msg=msg, rc=rc, stderr=stderr, stdout=stdout)
181
+        module.fail_json(msg=msg)
176 182
 
177 183
 
178 184
 def main():
@@ -184,6 +190,7 @@ def main():
184 190
             client_url=dict(),
185 191
             server_url=dict(),
186 192
             threads=dict(type='int', default=10),
193
+            processes=dict(type='int', default=1),
187 194
             duration=dict(type='int', default=10),
188 195
             timeout=dict(type='int', default=5),
189 196
             sending_delay=dict(type='float', default=-1.0),

+ 49
- 3
performa/scenarios/mq/omsimulator-threading.yaml View File

@@ -84,7 +84,7 @@ execution:
84 84
     tasks:
85 85
     - atop:
86 86
         command: stop
87
-        labels: [ PRC ]
87
+        labels: [ PRC, PRM ]
88 88
 
89 89
 aggregation:
90 90
   -
@@ -93,8 +93,54 @@ aggregation:
93 93
         { task: omsimulator }
94 94
       values:
95 95
         pipeline:
96
-        - { $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* } }}
97
-        - { $group: { _id: null, rabbit_sys: { $avg: "$sys" }, rabbit_user: { $avg: "$user" }, rabbit_total: { $avg: { $add: [ "$sys", "$user" ] }} }}
96
+        - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
97
+        - $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
98
+  -
99
+    update:
100
+      query:
101
+        { task: omsimulator }
102
+      values:
103
+        pipeline:
104
+        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
105
+        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
106
+
107
+{% if rabbit_hosts[1] %}
108
+  -
109
+    update:
110
+      query:
111
+        { task: omsimulator }
112
+      values:
113
+        pipeline:
114
+        - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
115
+        - $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
116
+  -
117
+    update:
118
+      query:
119
+        { task: omsimulator }
120
+      values:
121
+        pipeline:
122
+        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
123
+        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
124
+{% endif %}
125
+
126
+{% if rabbit_hosts[2] %}
127
+  -
128
+    update:
129
+      query:
130
+        { task: omsimulator }
131
+      values:
132
+        pipeline:
133
+        - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
134
+        - $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
135
+  -
136
+    update:
137
+      query:
138
+        { task: omsimulator }
139
+      values:
140
+        pipeline:
141
+        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
142
+        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
143
+{% endif %}
98 144
 
99 145
 report:
100 146
   template: omsimulator.rst

+ 265
- 55
performa/scenarios/mq/omsimulator.rst View File

@@ -8,11 +8,14 @@ with `Oslo.messaging Simulator`_
8 8
 Test Case 1: RPC CALL Throughput Test
9 9
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
10 10
 
11
-**Message processing**
11
+Message processing
12
+~~~~~~~~~~~~~~~~~~
12 13
 
13 14
 Messages are collected at 3 points: ``sent`` - messages sent by the client,
14 15
 ``received`` - messages received by the server, ``round-trip`` - replies
15 16
 received by the client. Also the number of lost messages is calculated.
17
+Sizes of messages is based on the distribution of messages collected on
18
+the 100-node cloud.
16 19
 
17 20
 {{'''
18 21
     title: RPC CALL Message count
@@ -25,7 +28,7 @@ received by the client. Also the number of lost messages is calculated.
25 28
     chart: line
26 29
     pipeline:
27 30
     - { $match: { task: omsimulator, mode: call }}
28
-    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
31
+    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } },
29 32
                   sent: { $sum: "$client.count" },
30 33
                   received: { $sum: "$server.count" },
31 34
                   round_trip: { $sum: "$round_trip.count" },
@@ -42,7 +45,8 @@ received by the client. Also the number of lost messages is calculated.
42 45
 }}
43 46
 
44 47
 
45
-**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
48
+The throughput, latency and RabbitMQ CPU utilization
49
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
46 50
 
47 51
 The chart shows the throughput, latency and CPU utilization by RabbitMQ server
48 52
 depending on number of concurrent threads.
@@ -58,22 +62,89 @@ depending on number of concurrent threads.
58 62
       y5: RabbitMQ CPU consumption, %
59 63
     chart: line
60 64
     pipeline:
61
-    - { $match: { task: omsimulator, mode: call }}
62
-    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
63
-                  msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }},
64
-                  msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }},
65
-                  msg_round_trip_per_sec: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }},
66
-                  latency: { $avg: "$round_trip.latency" },
67
-                  rabbit_total: { $avg: "$rabbit_total" }
68
-                }}
69
-    - { $project: { x: "$_id.threads",
70
-                    y: "$msg_sent_per_sec",
71
-                    y2: "$msg_received_per_sec",
72
-                    y3: "$msg_round_trip_per_sec",
73
-                    y4: { $multiply: [ "$latency", 1000 ] },
74
-                    y5: { $multiply: [ "$rabbit_total", 100 ] }
75
-                  }}
76
-    - { $sort: { x: 1 }}
65
+    - $match: { task: omsimulator, mode: call }
66
+    - $group:
67
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
68
+        msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
69
+        msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
70
+        msg_round_trip_per_sec: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }}
71
+        latency: { $avg: "$round_trip.latency" }
72
+        rabbit_total_0: { $avg: "$rabbit_total_0" }
73
+        rabbit_total_1: { $avg: "$rabbit_total_1" }
74
+        rabbit_total_2: { $avg: "$rabbit_total_2" }
75
+    - $project:
76
+        x: "$_id.threads"
77
+        y: "$msg_sent_per_sec"
78
+        y2: "$msg_received_per_sec"
79
+        y3: "$msg_round_trip_per_sec"
80
+        y4: { $multiply: [ "$latency", 1000 ] }
81
+        y5: { $multiply: [ { $sum: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
82
+''' | chart_and_table
83
+}}
84
+
85
+
86
+Detailed RabbitMQ CPU consumption
87
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
88
+
89
+Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
90
+
91
+{{'''
92
+    title: RabbitMQ nodes CPU consumption during RPC CALL load test
93
+    axes:
94
+      x: threads
95
+      y0: Master total, %
96
+      y1: Slave 1 total, %
97
+      y2: Slave 2 total, %
98
+      z0: Master sys, %
99
+      z1: Slave 1 sys, %
100
+      z2: Slave 2 sys, %
101
+    chart: line
102
+    pipeline:
103
+    - $match: { task: omsimulator, mode: call }
104
+    - $group:
105
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
106
+        rabbit_total_0: { $avg: "$rabbit_total_0" }
107
+        rabbit_total_1: { $avg: "$rabbit_total_1" }
108
+        rabbit_total_2: { $avg: "$rabbit_total_2" }
109
+        rabbit_sys_0: { $avg: "$rabbit_sys_0" }
110
+        rabbit_sys_1: { $avg: "$rabbit_sys_1" }
111
+        rabbit_sys_2: { $avg: "$rabbit_sys_2" }
112
+    - $project:
113
+        x: "$_id.threads"
114
+        y0: { $multiply: [ "$rabbit_total_0", 100 ] }
115
+        y1: { $multiply: [ "$rabbit_total_1", 100 ] }
116
+        y2: { $multiply: [ "$rabbit_total_2", 100 ] }
117
+        z0: { $multiply: [ "$rabbit_sys_0", 100 ] }
118
+        z1: { $multiply: [ "$rabbit_sys_1", 100 ] }
119
+        z2: { $multiply: [ "$rabbit_sys_2", 100 ] }
120
+''' | chart_and_table
121
+}}
122
+
123
+Detailed RabbitMQ resident memory
124
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
125
+
126
+Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
127
+
128
+{{'''
129
+    title: RabbitMQ nodes memory consumption during RPC CALL load test
130
+    axes:
131
+      x: threads
132
+      y0: Master, Mb
133
+      y1: Slave 1, Mb
134
+      y2: Slave 2, Mb
135
+    chart: line
136
+    pipeline:
137
+    - $match: { task: omsimulator, mode: call }
138
+    - $group:
139
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
140
+        rabbit_0: { $avg: "$rabbit_resident_0" }
141
+        rabbit_1: { $avg: "$rabbit_resident_1" }
142
+        rabbit_2: { $avg: "$rabbit_resident_2" }
143
+    - $project:
144
+        x: "$_id.threads"
145
+        y0: { $divide: [ "$rabbit_0", 1048576 ] }
146
+        y1: { $divide: [ "$rabbit_1", 1048576 ] }
147
+        y2: { $divide: [ "$rabbit_2", 1048576 ] }
77 148
 ''' | chart_and_table
78 149
 }}
79 150
 
@@ -81,11 +152,13 @@ depending on number of concurrent threads.
81 152
 Test Case 2: RPC CAST Throughput Test
82 153
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
83 154
 
84
-**Message processing**
155
+Message processing
156
+~~~~~~~~~~~~~~~~~~
85 157
 
86 158
 Messages are collected at 2 points: ``sent`` - messages sent by the client
87 159
 and ``received`` - messages received by the server. Also the number of lost
88
-messages is calculated.
160
+messages is calculated. Sizes of messages is based on the distribution of
161
+messages collected on the 100-node cloud.
89 162
 
90 163
 {{'''
91 164
     title: RPC CAST Message count
@@ -97,7 +170,7 @@ messages is calculated.
97 170
     chart: line
98 171
     pipeline:
99 172
     - { $match: { task: omsimulator, mode: cast }}
100
-    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
173
+    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } },
101 174
                   sent: { $sum: "$client.count" },
102 175
                   received: { $sum: "$server.count" },
103 176
                   lost: { $sum: { $subtract: ["$client.count", "$server.count"] }}
@@ -112,7 +185,8 @@ messages is calculated.
112 185
 }}
113 186
 
114 187
 
115
-**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
188
+The throughput, latency and RabbitMQ CPU utilization
189
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
116 190
 
117 191
 The chart shows the throughput, latency and CPU utilization by RabbitMQ server
118 192
 depending on number of concurrent threads.
@@ -127,20 +201,86 @@ depending on number of concurrent threads.
127 201
       y4: RabbitMQ CPU consumption, %
128 202
     chart: line
129 203
     pipeline:
130
-    - { $match: { task: omsimulator, mode: cast }}
131
-    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
132
-                  msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }},
133
-                  msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }},
134
-                  latency: { $avg: "$server.latency" },
135
-                  rabbit_total: { $avg: "$rabbit_total" }
136
-                }}
137
-    - { $project: { x: "$_id.threads",
138
-                    y: "$msg_sent_per_sec",
139
-                    y2: "$msg_received_per_sec",
140
-                    y3: { $multiply: [ "$latency", 1000 ] },
141
-                    y4: { $multiply: [ "$rabbit_total", 100 ] }
142
-                  }}
143
-    - { $sort: { x: 1 }}
204
+    - $match: { task: omsimulator, mode: cast }
205
+    - $group:
206
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
207
+        msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
208
+        msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
209
+        latency: { $avg: "$server.latency" }
210
+        rabbit_total_0: { $avg: "$rabbit_total_0" }
211
+        rabbit_total_1: { $avg: "$rabbit_total_1" }
212
+        rabbit_total_2: { $avg: "$rabbit_total_2" }
213
+    - $project:
214
+        x: "$_id.threads"
215
+        y: "$msg_sent_per_sec"
216
+        y2: "$msg_received_per_sec"
217
+        y3: { $multiply: [ "$latency", 1000 ] }
218
+        y4: { $multiply: [ { $sum: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
219
+''' | chart_and_table
220
+}}
221
+
222
+Detailed RabbitMQ CPU consumption
223
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
224
+
225
+Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
226
+
227
+{{'''
228
+    title: RabbitMQ nodes CPU consumption during RPC CAST load test
229
+    axes:
230
+      x: threads
231
+      y0: Master total, %
232
+      y1: Slave 1 total, %
233
+      y2: Slave 2 total, %
234
+      z0: Master sys, %
235
+      z1: Slave 1 sys, %
236
+      z2: Slave 2 sys, %
237
+    chart: line
238
+    pipeline:
239
+    - $match: { task: omsimulator, mode: cast }
240
+    - $group:
241
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
242
+        rabbit_total_0: { $avg: "$rabbit_total_0" }
243
+        rabbit_total_1: { $avg: "$rabbit_total_1" }
244
+        rabbit_total_2: { $avg: "$rabbit_total_2" }
245
+        rabbit_sys_0: { $avg: "$rabbit_sys_0" }
246
+        rabbit_sys_1: { $avg: "$rabbit_sys_1" }
247
+        rabbit_sys_2: { $avg: "$rabbit_sys_2" }
248
+    - $project:
249
+        x: "$_id.threads"
250
+        y0: { $multiply: [ "$rabbit_total_0", 100 ] }
251
+        y1: { $multiply: [ "$rabbit_total_1", 100 ] }
252
+        y2: { $multiply: [ "$rabbit_total_2", 100 ] }
253
+        z0: { $multiply: [ "$rabbit_sys_0", 100 ] }
254
+        z1: { $multiply: [ "$rabbit_sys_1", 100 ] }
255
+        z2: { $multiply: [ "$rabbit_sys_2", 100 ] }
256
+''' | chart_and_table
257
+}}
258
+
259
+Detailed RabbitMQ resident memory
260
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
261
+
262
+Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
263
+
264
+{{'''
265
+    title: RabbitMQ nodes memory consumption during RPC CAST load test
266
+    axes:
267
+      x: threads
268
+      y0: Master, Mb
269
+      y1: Slave 1, Mb
270
+      y2: Slave 2, Mb
271
+    chart: line
272
+    pipeline:
273
+    - $match: { task: omsimulator, mode: cast }
274
+    - $group:
275
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
276
+        rabbit_0: { $avg: "$rabbit_resident_0" }
277
+        rabbit_1: { $avg: "$rabbit_resident_1" }
278
+        rabbit_2: { $avg: "$rabbit_resident_2" }
279
+    - $project:
280
+        x: "$_id.threads"
281
+        y0: { $divide: [ "$rabbit_0", 1048576 ] }
282
+        y1: { $divide: [ "$rabbit_1", 1048576 ] }
283
+        y2: { $divide: [ "$rabbit_2", 1048576 ] }
144 284
 ''' | chart_and_table
145 285
 }}
146 286
 
@@ -148,11 +288,13 @@ depending on number of concurrent threads.
148 288
 Test Case 3: Notification Throughput Test
149 289
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
150 290
 
151
-**Message processing**
291
+Message processing
292
+~~~~~~~~~~~~~~~~~~
152 293
 
153 294
 Messages are collected at 2 points: ``sent`` - messages sent by the client
154 295
 and ``received`` - messages received by the server. Also the number of lost
155
-messages is calculated.
296
+messages is calculated. Sizes of messages is based on the distribution of
297
+messages collected on the 100-node cloud.
156 298
 
157 299
 {{'''
158 300
     title: NOTIFY Message count
@@ -164,7 +306,7 @@ messages is calculated.
164 306
     chart: line
165 307
     pipeline:
166 308
     - { $match: { task: omsimulator, mode: notify }}
167
-    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
309
+    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } },
168 310
                   sent: { $sum: "$client.count" },
169 311
                   received: { $sum: "$server.count" },
170 312
                   lost: { $sum: { $subtract: ["$client.count", "$server.count"] }}
@@ -179,7 +321,8 @@ messages is calculated.
179 321
 }}
180 322
 
181 323
 
182
-**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
324
+The throughput, latency and RabbitMQ CPU utilization
325
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
183 326
 
184 327
 The chart shows the throughput, latency and CPU utilization by RabbitMQ server
185 328
 depending on number of concurrent threads.
@@ -194,23 +337,90 @@ depending on number of concurrent threads.
194 337
       y4: RabbitMQ CPU consumption, %
195 338
     chart: line
196 339
     pipeline:
197
-    - { $match: { task: omsimulator, mode: notify }}
198
-    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
199
-                  msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }},
200
-                  msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }},
201
-                  latency: { $avg: "$server.latency" },
202
-                  rabbit_total: { $avg: "$rabbit_total" }
203
-                }}
204
-    - { $project: { x: "$_id.threads",
205
-                    y: "$msg_sent_per_sec",
206
-                    y2: "$msg_received_per_sec",
207
-                    y3: { $multiply: [ "$latency", 1000 ] },
208
-                    y4: { $multiply: [ "$rabbit_total", 100 ] }
209
-                  }}
210
-    - { $sort: { x: 1 }}
340
+    - $match: { task: omsimulator, mode: notify }
341
+    - $group:
342
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
343
+        msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
344
+        msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
345
+        latency: { $avg: "$server.latency" }
346
+        rabbit_total_0: { $avg: "$rabbit_total_0" }
347
+        rabbit_total_1: { $avg: "$rabbit_total_1" }
348
+        rabbit_total_2: { $avg: "$rabbit_total_2" }
349
+    - $project:
350
+        x: "$_id.threads"
351
+        y: "$msg_sent_per_sec"
352
+        y2: "$msg_received_per_sec"
353
+        y3: { $multiply: [ "$latency", 1000 ] }
354
+        y4: { $multiply: [ { $sum: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
355
+''' | chart_and_table
356
+}}
357
+
358
+Detailed RabbitMQ CPU consumption
359
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
360
+
361
+Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
362
+
363
+{{'''
364
+    title: RabbitMQ nodes CPU consumption during NOTIFY load test
365
+    axes:
366
+      x: threads
367
+      y0: Master total, %
368
+      y1: Slave 1 total, %
369
+      y2: Slave 2 total, %
370
+      z0: Master sys, %
371
+      z1: Slave 1 sys, %
372
+      z2: Slave 2 sys, %
373
+    chart: line
374
+    pipeline:
375
+    - $match: { task: omsimulator, mode: notify }
376
+    - $group:
377
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
378
+        rabbit_total_0: { $avg: "$rabbit_total_0" }
379
+        rabbit_total_1: { $avg: "$rabbit_total_1" }
380
+        rabbit_total_2: { $avg: "$rabbit_total_2" }
381
+        rabbit_sys_0: { $avg: "$rabbit_sys_0" }
382
+        rabbit_sys_1: { $avg: "$rabbit_sys_1" }
383
+        rabbit_sys_2: { $avg: "$rabbit_sys_2" }
384
+    - $project:
385
+        x: "$_id.threads"
386
+        y0: { $multiply: [ "$rabbit_total_0", 100 ] }
387
+        y1: { $multiply: [ "$rabbit_total_1", 100 ] }
388
+        y2: { $multiply: [ "$rabbit_total_2", 100 ] }
389
+        z0: { $multiply: [ "$rabbit_sys_0", 100 ] }
390
+        z1: { $multiply: [ "$rabbit_sys_1", 100 ] }
391
+        z2: { $multiply: [ "$rabbit_sys_2", 100 ] }
211 392
 ''' | chart_and_table
212 393
 }}
213 394
 
395
+Detailed RabbitMQ resident memory
396
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
397
+
398
+Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
399
+
400
+{{'''
401
+    title: RabbitMQ nodes memory consumption during NOTIFY load test
402
+    axes:
403
+      x: threads
404
+      y0: Master, Mb
405
+      y1: Slave 1, Mb
406
+      y2: Slave 2, Mb
407
+    chart: line
408
+    pipeline:
409
+    - $match: { task: omsimulator, mode: notify }
410
+    - $group:
411
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
412
+        rabbit_0: { $avg: "$rabbit_resident_0" }
413
+        rabbit_1: { $avg: "$rabbit_resident_1" }
414
+        rabbit_2: { $avg: "$rabbit_resident_2" }
415
+    - $project:
416
+        x: "$_id.threads"
417
+        y0: { $divide: [ "$rabbit_0", 1048576 ] }
418
+        y1: { $divide: [ "$rabbit_1", 1048576 ] }
419
+        y2: { $divide: [ "$rabbit_2", 1048576 ] }
420
+''' | chart_and_table
421
+}}
422
+
423
+
214 424
 .. references:
215 425
 
216 426
 .. _message_queue_performance: http://docs.openstack.org/developer/performance-docs/test_plans/mq/plan.html

+ 52
- 6
performa/scenarios/mq/omsimulator.yaml View File

@@ -49,7 +49,7 @@ execution:
49 49
   -
50 50
     hosts: {{ tester_hosts }}
51 51
     matrix:
52
-      host_count: [ 1, 2, 5, 10 ]
52
+      processes: [ 1, 2, 5, 7, 10 ]
53 53
     tasks:
54 54
     - omsimulator:
55 55
         mode: call
@@ -61,7 +61,7 @@ execution:
61 61
   -
62 62
     hosts: {{ tester_hosts }}
63 63
     matrix:
64
-      host_count: [ 1, 2, 5, 10 ]
64
+      processes: [ 1, 2, 5, 7, 10 ]
65 65
     tasks:
66 66
     - omsimulator:
67 67
         mode: cast
@@ -73,7 +73,7 @@ execution:
73 73
   -
74 74
     hosts: {{ tester_hosts }}
75 75
     matrix:
76
-      host_count: [ 1, 2, 5, 10 ]
76
+      processes: [ 1, 2, 5, 7, 10 ]
77 77
     tasks:
78 78
     - omsimulator:
79 79
         mode: notify
@@ -87,7 +87,7 @@ execution:
87 87
     tasks:
88 88
     - atop:
89 89
         command: stop
90
-        labels: [ PRC ]
90
+        labels: [ PRC, PRM ]
91 91
 
92 92
 aggregation:
93 93
   -
@@ -96,8 +96,54 @@ aggregation:
96 96
         { task: omsimulator }
97 97
       values:
98 98
         pipeline:
99
-        - { $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* } }}
100
-        - { $group: { _id: null, rabbit_sys: { $avg: "$sys" }, rabbit_user: { $avg: "$user" }, rabbit_total: { $avg: { $add: [ "$sys", "$user" ] }} }}
99
+        - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
100
+        - $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
101
+  -
102
+    update:
103
+      query:
104
+        { task: omsimulator }
105
+      values:
106
+        pipeline:
107
+        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
108
+        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
109
+
110
+{% if rabbit_hosts[1] %}
111
+  -
112
+    update:
113
+      query:
114
+        { task: omsimulator }
115
+      values:
116
+        pipeline:
117
+        - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
118
+        - $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
119
+  -
120
+    update:
121
+      query:
122
+        { task: omsimulator }
123
+      values:
124
+        pipeline:
125
+        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
126
+        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
127
+{% endif %}
128
+
129
+{% if rabbit_hosts[2] %}
130
+  -
131
+    update:
132
+      query:
133
+        { task: omsimulator }
134
+      values:
135
+        pipeline:
136
+        - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
137
+        - $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
138
+  -
139
+    update:
140
+      query:
141
+        { task: omsimulator }
142
+      values:
143
+        pipeline:
144
+        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
145
+        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
146
+{% endif %}
101 147
 
102 148
 report:
103 149
   template: omsimulator.rst

Loading…
Cancel
Save