Run multiple omsimulator pairs on a single node

performa/modules/omsimulator.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python
 import copy
+import multiprocessing
 import os
 import signal
 import tempfile
@@ -11,35 +12,6 @@ UNIQUE_NAME = 'performa_omsimulator'
 DIR = '/tmp/performa/oslo.messaging/tools/'
 PYTHON = '/tmp/performa/oslo.messaging/.venv/bin/python'
 
-PATTERNS = [
-    r'(?P<msg_sent>\d+) messages were sent for (?P<duration>\d+) sec',
-    r'(?P<bytes_sent>\d+) bytes were sent for',
-]
-TRANSFORM_FIELDS = {
-    'msg_sent': int,
-    'bytes_sent': int,
-    'duration': int,
-}
-
-
-def parse_output(raw):
-    result = {}
-
-    for pattern in PATTERNS:
-        for parsed in re.finditer(pattern, raw):
-            result.update(parsed.groupdict())
-
-    for k in result.keys():
-        if k in TRANSFORM_FIELDS:
-            result[k] = TRANSFORM_FIELDS[k](result[k])
-
-    result['msg_sent_bandwidth'] = (result.get('msg_sent', 0) /
-                                    result.get('duration', 1))
-    result['bytes_sent_bandwidth'] = (result.get('bytes_sent', 0) /
-                                      result.get('duration', 1))
-
-    return result
-
 
 def chdir(module):
     try:
@@ -48,29 +20,6 @@ def chdir(module):
         module.fail_json(msg='Failed to change dir to %s: %s' % (DIR, e))
 
 
-def start_daemon(module, cmd):
-    cmd = ('daemon -n %(name)s -D %(dir)s -F %(pid)s -U -- %(cmd)s' %
-           dict(name=UNIQUE_NAME, dir=DIR, pid=SERVER_PID, cmd=cmd))
-
-    rc, stdout, stderr = module.run_command(cmd)
-    result = dict(changed=True, rc=rc, stdout=stdout, stderr=stderr, cmd=cmd)
-
-    if rc:
-        module.fail_json(msg='Failed to start omsimulator', **result)
-
-
-def stop_daemon(module):
-    rc, stdout, stderr = module.run_command('/bin/cat %s' % SERVER_PID)
-
-    if rc:
-        return
-
-    rc, stdout, stderr = module.run_command('pgrep -P %s' % stdout)
-    os.kill(int(stdout), signal.SIGINT)
-
-    time.sleep(2)
-
-
 def read_file(filename):
     fd = None
     try:
@@ -92,35 +41,16 @@ def transform_series(series):
     return result
 
 
-def run(module):
+def make_file_name(base, index):
-    params = copy.deepcopy(module.params)
+    return '%s-%s' % (base, index)
 
-    if params['mode'] == 'notify':
-        server_tool = 'notify-server'
-        client_tool = 'notify-client'
-    else:
-        server_tool = 'rpc-server'
-        client_tool = 'rpc-client'
-
-    params['python'] = PYTHON
-    # todo: fix topic support in omsimulator
-    # params['topic'] = 'performa-%d' % (random.random() * 1000000)
-    params['server_tool'] = server_tool
-    params['client_tool'] = client_tool
-    params['server_file'] = SERVER_FILE_NAME
-    params['client_file'] = CLIENT_FILE_NAME
-
-    params['url'] = params['server_url'] or params['url']
-    server = ('%(python)s simulator.py '
-              # '--topic %(topic)s '
-              '--url %(url)s '
-              '--json %(server_file)s '
-              '%(server_tool)s ') % params
 
+def make_client_cmd(params, i):
+    params['client_file'] = make_file_name(CLIENT_FILE_NAME, i)
     params['url'] = params['client_url'] or params['url']
+
     client = ('%(python)s simulator.py '
-              # '--topic %(topic)s '
+              '--url %(url)s '
-              '--url=%(url)s '
               '--json %(client_file)s '
               '-l %(duration)s '
               '%(client_tool)s '
@@ -134,45 +64,121 @@ def run(module):
     if params['mode'] == 'fanout':
         client += '--is-fanout True '
 
-    start_daemon(module, server)
+    return client
 
-    rc, stdout, stderr = module.run_command(client)
 
-    if rc:
+def make_server_cmd(params, i):
-        module.fail_json(msg='Failed to start omsimulator',
+    params['server_file'] = make_file_name(SERVER_FILE_NAME, i)
-                         stderr=stderr, rc=rc, cmd=client)
+    params['url'] = params['server_url'] or params['url']
 
-    stop_daemon(module)
+    server = ('%(python)s simulator.py '
+              '--url %(url)s '
+              '--json %(server_file)s '
+              '%(server_tool)s ') % params
 
-    try:
+    return server
-        client_data = read_file(CLIENT_FILE_NAME)
+
-        server_data = read_file(SERVER_FILE_NAME)
+
+def run_client(module, command):
+    module.run_command(command)
+
+
+def run_client_pool(module, params):
+    processes = []
+
+    for i in range(params['processes']):
+        cmd = make_client_cmd(params, i)
+        p = multiprocessing.Process(target=run_client, args=(module, cmd))
+        processes.append(p)
+        p.start()
+
+    for p in processes:
+        p.join()
+
+
+def run_server(module, command):
+    module.run_command(command)
+
+
+def start_server_pool(module, params):
+    processes = []
+
+    for i in range(params['processes']):
+        cmd = make_server_cmd(params, i)
+        p = multiprocessing.Process(target=run_client, args=(module, cmd))
+        processes.append(p)
+        p.start()
+
+    return processes
+
+
+def stop_server_pool(module, processes):
+    for p in processes:
+        rc, stdout, stderr = module.run_command('pgrep -P %s' % p.pid)
+
+        for child in (int(p) for p in stdout.split('\n') if p):
+            os.kill(child, signal.SIGINT)
+
+    time.sleep(3)  # let simulators handle the signal
+
+    for p in processes:
+        os.kill(p.pid, signal.SIGINT)
+        p.join()
+
+
+def collect_results(params):
+    result = dict(records=[], series=[])
+
+    for i in range(params['processes']):
+        client_data = read_file(make_file_name(CLIENT_FILE_NAME, i))
+        server_data = read_file(make_file_name(SERVER_FILE_NAME, i))
 
         client_summary = client_data['summary']['client']
-
+        record = dict(start=client_summary['start'], end=client_summary['end'],
-        record = dict(start=client_summary['start'],
-                      end=client_summary['end'],
                       client=client_summary)
-
         if 'round_trip' in client_data['summary']:
             round_trip_summary = client_data['summary']['round_trip']
             record['round_trip'] = round_trip_summary
-
         if 'error' in client_data['summary']:
             error_summary = client_data['summary']['error']
             record['error'] = error_summary
-
         server_summary = server_data['summary']
         record['server'] = server_summary
-
         series = transform_series(client_data['series'])
         series.extend(transform_series(server_data['series']))
 
-        result = dict(records=[record], series=series)
+        result['records'].append(record)
+        result['series'] += series
+
+    return result
+
+
+def run(module):
+    params = copy.deepcopy(module.params)
+
+    if params['mode'] == 'notify':
+        server_tool = 'notify-server'
+        client_tool = 'notify-client'
+    else:
+        server_tool = 'rpc-server'
+        client_tool = 'rpc-client'
+
+    params['python'] = PYTHON
+    params['server_tool'] = server_tool
+    params['client_tool'] = client_tool
+
+    server_processes = start_server_pool(module, params)
+
+    run_client_pool(module, params)
+
+    stop_server_pool(module, server_processes)
+
+    try:
+        result = collect_results(params)
         module.exit_json(**result)
     except Exception as e:
         msg = 'Failed to read omsimulator output: %s' % e
-        module.fail_json(msg=msg, rc=rc, stderr=stderr, stdout=stdout)
+        module.fail_json(msg=msg)
 
 
 def main():
@@ -184,6 +190,7 @@ def main():
             client_url=dict(),
             server_url=dict(),
             threads=dict(type='int', default=10),
+            processes=dict(type='int', default=1),
             duration=dict(type='int', default=10),
             timeout=dict(type='int', default=5),
             sending_delay=dict(type='float', default=-1.0),

performa/scenarios/mq/omsimulator-threading.yaml

@@ -84,7 +84,7 @@ execution:
     tasks:
     - atop:
         command: stop
-        labels: [ PRC ]
+        labels: [ PRC, PRM ]
 
 aggregation:
   -
@@ -93,8 +93,54 @@ aggregation:
         { task: omsimulator }
       values:
         pipeline:
-        - { $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* } }}
+        - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
-        - { $group: { _id: null, rabbit_sys: { $avg: "$sys" }, rabbit_user: { $avg: "$user" }, rabbit_total: { $avg: { $add: [ "$sys", "$user" ] }} }}
+        - $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
+  -
+    update:
+      query:
+        { task: omsimulator }
+      values:
+        pipeline:
+        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
+        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
+
+{% if rabbit_hosts[1] %}
+  -
+    update:
+      query:
+        { task: omsimulator }
+      values:
+        pipeline:
+        - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
+        - $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
+  -
+    update:
+      query:
+        { task: omsimulator }
+      values:
+        pipeline:
+        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
+        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
+{% endif %}
+
+{% if rabbit_hosts[2] %}
+  -
+    update:
+      query:
+        { task: omsimulator }
+      values:
+        pipeline:
+        - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
+        - $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
+  -
+    update:
+      query:
+        { task: omsimulator }
+      values:
+        pipeline:
+        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
+        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
+{% endif %}
 
 report:
   template: omsimulator.rst

performa/scenarios/mq/omsimulator.rst

@@ -8,11 +8,14 @@ with `Oslo.messaging Simulator`_
 Test Case 1: RPC CALL Throughput Test
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-**Message processing**
+Message processing
+~~~~~~~~~~~~~~~~~~
 
 Messages are collected at 3 points: ``sent`` - messages sent by the client,
 ``received`` - messages received by the server, ``round-trip`` - replies
 received by the client. Also the number of lost messages is calculated.
+Sizes of messages is based on the distribution of messages collected on
+the 100-node cloud.
 
 {{'''
     title: RPC CALL Message count
@@ -25,7 +28,7 @@ received by the client. Also the number of lost messages is calculated.
     chart: line
     pipeline:
     - { $match: { task: omsimulator, mode: call }}
-    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
+    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } },
                   sent: { $sum: "$client.count" },
                   received: { $sum: "$server.count" },
                   round_trip: { $sum: "$round_trip.count" },
@@ -42,7 +45,8 @@ received by the client. Also the number of lost messages is calculated.
 }}
 
 
-**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
+The throughput, latency and RabbitMQ CPU utilization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The chart shows the throughput, latency and CPU utilization by RabbitMQ server
 depending on number of concurrent threads.
@@ -58,22 +62,89 @@ depending on number of concurrent threads.
       y5: RabbitMQ CPU consumption, %
     chart: line
     pipeline:
-    - { $match: { task: omsimulator, mode: call }}
+    - $match: { task: omsimulator, mode: call }
-    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
+    - $group:
-                  msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }},
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
-                  msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }},
+        msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
-                  msg_round_trip_per_sec: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }},
+        msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
-                  latency: { $avg: "$round_trip.latency" },
+        msg_round_trip_per_sec: { $sum: { $divide: ["$round_trip.count", "$round_trip.duration"] }}
-                  rabbit_total: { $avg: "$rabbit_total" }
+        latency: { $avg: "$round_trip.latency" }
-                }}
+        rabbit_total_0: { $avg: "$rabbit_total_0" }
-    - { $project: { x: "$_id.threads",
+        rabbit_total_1: { $avg: "$rabbit_total_1" }
-                    y: "$msg_sent_per_sec",
+        rabbit_total_2: { $avg: "$rabbit_total_2" }
-                    y2: "$msg_received_per_sec",
+    - $project:
-                    y3: "$msg_round_trip_per_sec",
+        x: "$_id.threads"
-                    y4: { $multiply: [ "$latency", 1000 ] },
+        y: "$msg_sent_per_sec"
-                    y5: { $multiply: [ "$rabbit_total", 100 ] }
+        y2: "$msg_received_per_sec"
-                  }}
+        y3: "$msg_round_trip_per_sec"
-    - { $sort: { x: 1 }}
+        y4: { $multiply: [ "$latency", 1000 ] }
+        y5: { $multiply: [ { $sum: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
+''' | chart_and_table
+}}
+
+
+Detailed RabbitMQ CPU consumption
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
+
+{{'''
+    title: RabbitMQ nodes CPU consumption during RPC CALL load test
+    axes:
+      x: threads
+      y0: Master total, %
+      y1: Slave 1 total, %
+      y2: Slave 2 total, %
+      z0: Master sys, %
+      z1: Slave 1 sys, %
+      z2: Slave 2 sys, %
+    chart: line
+    pipeline:
+    - $match: { task: omsimulator, mode: call }
+    - $group:
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
+        rabbit_total_0: { $avg: "$rabbit_total_0" }
+        rabbit_total_1: { $avg: "$rabbit_total_1" }
+        rabbit_total_2: { $avg: "$rabbit_total_2" }
+        rabbit_sys_0: { $avg: "$rabbit_sys_0" }
+        rabbit_sys_1: { $avg: "$rabbit_sys_1" }
+        rabbit_sys_2: { $avg: "$rabbit_sys_2" }
+    - $project:
+        x: "$_id.threads"
+        y0: { $multiply: [ "$rabbit_total_0", 100 ] }
+        y1: { $multiply: [ "$rabbit_total_1", 100 ] }
+        y2: { $multiply: [ "$rabbit_total_2", 100 ] }
+        z0: { $multiply: [ "$rabbit_sys_0", 100 ] }
+        z1: { $multiply: [ "$rabbit_sys_1", 100 ] }
+        z2: { $multiply: [ "$rabbit_sys_2", 100 ] }
+''' | chart_and_table
+}}
+
+Detailed RabbitMQ resident memory
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
+
+{{'''
+    title: RabbitMQ nodes memory consumption during RPC CALL load test
+    axes:
+      x: threads
+      y0: Master, Mb
+      y1: Slave 1, Mb
+      y2: Slave 2, Mb
+    chart: line
+    pipeline:
+    - $match: { task: omsimulator, mode: call }
+    - $group:
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
+        rabbit_0: { $avg: "$rabbit_resident_0" }
+        rabbit_1: { $avg: "$rabbit_resident_1" }
+        rabbit_2: { $avg: "$rabbit_resident_2" }
+    - $project:
+        x: "$_id.threads"
+        y0: { $divide: [ "$rabbit_0", 1048576 ] }
+        y1: { $divide: [ "$rabbit_1", 1048576 ] }
+        y2: { $divide: [ "$rabbit_2", 1048576 ] }
 ''' | chart_and_table
 }}
 
@@ -81,11 +152,13 @@ depending on number of concurrent threads.
 Test Case 2: RPC CAST Throughput Test
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-**Message processing**
+Message processing
+~~~~~~~~~~~~~~~~~~
 
 Messages are collected at 2 points: ``sent`` - messages sent by the client
 and ``received`` - messages received by the server. Also the number of lost
-messages is calculated.
+messages is calculated. Sizes of messages is based on the distribution of
+messages collected on the 100-node cloud.
 
 {{'''
     title: RPC CAST Message count
@@ -97,7 +170,7 @@ messages is calculated.
     chart: line
     pipeline:
     - { $match: { task: omsimulator, mode: cast }}
-    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
+    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } },
                   sent: { $sum: "$client.count" },
                   received: { $sum: "$server.count" },
                   lost: { $sum: { $subtract: ["$client.count", "$server.count"] }}
@@ -112,7 +185,8 @@ messages is calculated.
 }}
 
 
-**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
+The throughput, latency and RabbitMQ CPU utilization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The chart shows the throughput, latency and CPU utilization by RabbitMQ server
 depending on number of concurrent threads.
@@ -127,20 +201,86 @@ depending on number of concurrent threads.
       y4: RabbitMQ CPU consumption, %
     chart: line
     pipeline:
-    - { $match: { task: omsimulator, mode: cast }}
+    - $match: { task: omsimulator, mode: cast }
-    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
+    - $group:
-                  msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }},
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
-                  msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }},
+        msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
-                  latency: { $avg: "$server.latency" },
+        msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
-                  rabbit_total: { $avg: "$rabbit_total" }
+        latency: { $avg: "$server.latency" }
-                }}
+        rabbit_total_0: { $avg: "$rabbit_total_0" }
-    - { $project: { x: "$_id.threads",
+        rabbit_total_1: { $avg: "$rabbit_total_1" }
-                    y: "$msg_sent_per_sec",
+        rabbit_total_2: { $avg: "$rabbit_total_2" }
-                    y2: "$msg_received_per_sec",
+    - $project:
-                    y3: { $multiply: [ "$latency", 1000 ] },
+        x: "$_id.threads"
-                    y4: { $multiply: [ "$rabbit_total", 100 ] }
+        y: "$msg_sent_per_sec"
-                  }}
+        y2: "$msg_received_per_sec"
-    - { $sort: { x: 1 }}
+        y3: { $multiply: [ "$latency", 1000 ] }
+        y4: { $multiply: [ { $sum: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
+''' | chart_and_table
+}}
+
+Detailed RabbitMQ CPU consumption
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
+
+{{'''
+    title: RabbitMQ nodes CPU consumption during RPC CAST load test
+    axes:
+      x: threads
+      y0: Master total, %
+      y1: Slave 1 total, %
+      y2: Slave 2 total, %
+      z0: Master sys, %
+      z1: Slave 1 sys, %
+      z2: Slave 2 sys, %
+    chart: line
+    pipeline:
+    - $match: { task: omsimulator, mode: cast }
+    - $group:
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
+        rabbit_total_0: { $avg: "$rabbit_total_0" }
+        rabbit_total_1: { $avg: "$rabbit_total_1" }
+        rabbit_total_2: { $avg: "$rabbit_total_2" }
+        rabbit_sys_0: { $avg: "$rabbit_sys_0" }
+        rabbit_sys_1: { $avg: "$rabbit_sys_1" }
+        rabbit_sys_2: { $avg: "$rabbit_sys_2" }
+    - $project:
+        x: "$_id.threads"
+        y0: { $multiply: [ "$rabbit_total_0", 100 ] }
+        y1: { $multiply: [ "$rabbit_total_1", 100 ] }
+        y2: { $multiply: [ "$rabbit_total_2", 100 ] }
+        z0: { $multiply: [ "$rabbit_sys_0", 100 ] }
+        z1: { $multiply: [ "$rabbit_sys_1", 100 ] }
+        z2: { $multiply: [ "$rabbit_sys_2", 100 ] }
+''' | chart_and_table
+}}
+
+Detailed RabbitMQ resident memory
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
+
+{{'''
+    title: RabbitMQ nodes memory consumption during RPC CAST load test
+    axes:
+      x: threads
+      y0: Master, Mb
+      y1: Slave 1, Mb
+      y2: Slave 2, Mb
+    chart: line
+    pipeline:
+    - $match: { task: omsimulator, mode: cast }
+    - $group:
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
+        rabbit_0: { $avg: "$rabbit_resident_0" }
+        rabbit_1: { $avg: "$rabbit_resident_1" }
+        rabbit_2: { $avg: "$rabbit_resident_2" }
+    - $project:
+        x: "$_id.threads"
+        y0: { $divide: [ "$rabbit_0", 1048576 ] }
+        y1: { $divide: [ "$rabbit_1", 1048576 ] }
+        y2: { $divide: [ "$rabbit_2", 1048576 ] }
 ''' | chart_and_table
 }}
 
@@ -148,11 +288,13 @@ depending on number of concurrent threads.
 Test Case 3: Notification Throughput Test
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-**Message processing**
+Message processing
+~~~~~~~~~~~~~~~~~~
 
 Messages are collected at 2 points: ``sent`` - messages sent by the client
 and ``received`` - messages received by the server. Also the number of lost
-messages is calculated.
+messages is calculated. Sizes of messages is based on the distribution of
+messages collected on the 100-node cloud.
 
 {{'''
     title: NOTIFY Message count
@@ -164,7 +306,7 @@ messages is calculated.
     chart: line
     pipeline:
     - { $match: { task: omsimulator, mode: notify }}
-    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
+    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } },
                   sent: { $sum: "$client.count" },
                   received: { $sum: "$server.count" },
                   lost: { $sum: { $subtract: ["$client.count", "$server.count"] }}
@@ -179,7 +321,8 @@ messages is calculated.
 }}
 
 
-**Message throughput, latency and RabbitMQ CPU utilization depending on thread count**
+The throughput, latency and RabbitMQ CPU utilization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The chart shows the throughput, latency and CPU utilization by RabbitMQ server
 depending on number of concurrent threads.
@@ -194,23 +337,90 @@ depending on number of concurrent threads.
       y4: RabbitMQ CPU consumption, %
     chart: line
     pipeline:
-    - { $match: { task: omsimulator, mode: notify }}
+    - $match: { task: omsimulator, mode: notify }
-    - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } },
+    - $group:
-                  msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }},
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
-                  msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }},
+        msg_sent_per_sec: { $sum: { $divide: ["$client.count", "$client.duration"] }}
-                  latency: { $avg: "$server.latency" },
+        msg_received_per_sec: { $sum: { $divide: ["$server.count", "$server.duration"] }}
-                  rabbit_total: { $avg: "$rabbit_total" }
+        latency: { $avg: "$server.latency" }
-                }}
+        rabbit_total_0: { $avg: "$rabbit_total_0" }
-    - { $project: { x: "$_id.threads",
+        rabbit_total_1: { $avg: "$rabbit_total_1" }
-                    y: "$msg_sent_per_sec",
+        rabbit_total_2: { $avg: "$rabbit_total_2" }
-                    y2: "$msg_received_per_sec",
+    - $project:
-                    y3: { $multiply: [ "$latency", 1000 ] },
+        x: "$_id.threads"
-                    y4: { $multiply: [ "$rabbit_total", 100 ] }
+        y: "$msg_sent_per_sec"
-                  }}
+        y2: "$msg_received_per_sec"
-    - { $sort: { x: 1 }}
+        y3: { $multiply: [ "$latency", 1000 ] }
+        y4: { $multiply: [ { $sum: ["$rabbit_total_0", "$rabbit_total_1", "$rabbit_total_2"] }, 100 ] }
 ''' | chart_and_table
 }}
 
+Detailed RabbitMQ CPU consumption
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Thus chart shows statistics on RabbitMQ CPU consumption per nodes.
+
+{{'''
+    title: RabbitMQ nodes CPU consumption during NOTIFY load test
+    axes:
+      x: threads
+      y0: Master total, %
+      y1: Slave 1 total, %
+      y2: Slave 2 total, %
+      z0: Master sys, %
+      z1: Slave 1 sys, %
+      z2: Slave 2 sys, %
+    chart: line
+    pipeline:
+    - $match: { task: omsimulator, mode: notify }
+    - $group:
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
+        rabbit_total_0: { $avg: "$rabbit_total_0" }
+        rabbit_total_1: { $avg: "$rabbit_total_1" }
+        rabbit_total_2: { $avg: "$rabbit_total_2" }
+        rabbit_sys_0: { $avg: "$rabbit_sys_0" }
+        rabbit_sys_1: { $avg: "$rabbit_sys_1" }
+        rabbit_sys_2: { $avg: "$rabbit_sys_2" }
+    - $project:
+        x: "$_id.threads"
+        y0: { $multiply: [ "$rabbit_total_0", 100 ] }
+        y1: { $multiply: [ "$rabbit_total_1", 100 ] }
+        y2: { $multiply: [ "$rabbit_total_2", 100 ] }
+        z0: { $multiply: [ "$rabbit_sys_0", 100 ] }
+        z1: { $multiply: [ "$rabbit_sys_1", 100 ] }
+        z2: { $multiply: [ "$rabbit_sys_2", 100 ] }
+''' | chart_and_table
+}}
+
+Detailed RabbitMQ resident memory
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Thus chart shows statistics on RabbitMQ memory consumption (RSS) per nodes.
+
+{{'''
+    title: RabbitMQ nodes memory consumption during NOTIFY load test
+    axes:
+      x: threads
+      y0: Master, Mb
+      y1: Slave 1, Mb
+      y2: Slave 2, Mb
+    chart: line
+    pipeline:
+    - $match: { task: omsimulator, mode: notify }
+    - $group:
+        _id: { threads: { $multiply: [ "$threads", "$host_count", "$processes" ] } }
+        rabbit_0: { $avg: "$rabbit_resident_0" }
+        rabbit_1: { $avg: "$rabbit_resident_1" }
+        rabbit_2: { $avg: "$rabbit_resident_2" }
+    - $project:
+        x: "$_id.threads"
+        y0: { $divide: [ "$rabbit_0", 1048576 ] }
+        y1: { $divide: [ "$rabbit_1", 1048576 ] }
+        y2: { $divide: [ "$rabbit_2", 1048576 ] }
+''' | chart_and_table
+}}
+
+
 .. references:
 
 .. _message_queue_performance: http://docs.openstack.org/developer/performance-docs/test_plans/mq/plan.html

performa/scenarios/mq/omsimulator.yaml

@@ -49,7 +49,7 @@ execution:
   -
     hosts: {{ tester_hosts }}
     matrix:
-      host_count: [ 1, 2, 5, 10 ]
+      processes: [ 1, 2, 5, 7, 10 ]
     tasks:
     - omsimulator:
         mode: call
@@ -61,7 +61,7 @@ execution:
   -
     hosts: {{ tester_hosts }}
     matrix:
-      host_count: [ 1, 2, 5, 10 ]
+      processes: [ 1, 2, 5, 7, 10 ]
     tasks:
     - omsimulator:
         mode: cast
@@ -73,7 +73,7 @@ execution:
   -
     hosts: {{ tester_hosts }}
     matrix:
-      host_count: [ 1, 2, 5, 10 ]
+      processes: [ 1, 2, 5, 7, 10 ]
     tasks:
     - omsimulator:
         mode: notify
@@ -87,7 +87,7 @@ execution:
     tasks:
     - atop:
         command: stop
-        labels: [ PRC ]
+        labels: [ PRC, PRM ]
 
 aggregation:
   -
@@ -96,8 +96,54 @@ aggregation:
         { task: omsimulator }
       values:
         pipeline:
-        - { $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* } }}
+        - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
-        - { $group: { _id: null, rabbit_sys: { $avg: "$sys" }, rabbit_user: { $avg: "$user" }, rabbit_total: { $avg: { $add: [ "$sys", "$user" ] }} }}
+        - $group: { _id: null, rabbit_sys_0: { $avg: "$sys" }, rabbit_user_0: { $avg: "$user" }, rabbit_total_0: { $avg: { $add: [ "$sys", "$user" ] }} }
+  -
+    update:
+      query:
+        { task: omsimulator }
+      values:
+        pipeline:
+        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[0] }} }
+        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
+
+{% if rabbit_hosts[1] %}
+  -
+    update:
+      query:
+        { task: omsimulator }
+      values:
+        pipeline:
+        - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
+        - $group: { _id: null, rabbit_sys_1: { $avg: "$sys" }, rabbit_user_1: { $avg: "$user" }, rabbit_total_1: { $avg: { $add: [ "$sys", "$user" ] }} }
+  -
+    update:
+      query:
+        { task: omsimulator }
+      values:
+        pipeline:
+        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[1] }} }
+        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
+{% endif %}
+
+{% if rabbit_hosts[2] %}
+  -
+    update:
+      query:
+        { task: omsimulator }
+      values:
+        pipeline:
+        - $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
+        - $group: { _id: null, rabbit_sys_2: { $avg: "$sys" }, rabbit_user_2: { $avg: "$user" }, rabbit_total_2: { $avg: { $add: [ "$sys", "$user" ] }} }
+  -
+    update:
+      query:
+        { task: omsimulator }
+      values:
+        pipeline:
+        - $match: { task: atop, status: OK, label: PRM, name: { $regex: beam.* }, host: {{ rabbit_hosts[2] }} }
+        - $group: { _id: null, rabbit_resident_0: { $avg: "$resident" } }
+{% endif %}
 
 report:
   template: omsimulator.rst