Multicharts and failover reports

This commit is contained in:
Ilya Shakhat 2016-03-21 13:32:03 +03:00
parent d5e8cfc269
commit c33edfecb5
7 changed files with 195 additions and 59 deletions

View File

@ -33,24 +33,45 @@ from performa.engine import utils
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
def generate_chart(chart_str, db, doc_folder, tag): def generate_chart(chart_str, db, doc_folder, tag,
show_chart=True, show_table=True):
chart = yaml.safe_load(chart_str) chart = yaml.safe_load(chart_str)
pipeline = chart.get('pipeline') pipeline = chart.get('pipeline') # single pipeline
pipelines = chart.get('pipelines') # multiple pipelines
title = chart.get('title') title = chart.get('title')
fill = chart.get('fill') or False fill = chart.get('fill') or False
axes = chart.get('axes') or dict(x='x', y='y') axes = chart.get('axes') or dict(x='x', y='y')
do_round = show_table
collection_name = chart.get('collection') or 'records' collection_name = chart.get('collection') or 'records'
collection = db.get_collection(collection_name) collection = db.get_collection(collection_name)
LOG.debug('Title: %s', title) LOG.debug('Title: %s', title)
pipeline.insert(0, {'$match': {'status': 'OK'}}) axes_keys = sorted(axes.keys())
y_keys = set(axes.keys()) ^ set('x')
if tag: chart_data = collections.defaultdict(dict)
pipeline.insert(0, {'$match': {'tag': tag}})
chart_data = collection.aggregate(pipeline) for pl in (pipelines or [pipeline]):
pl.insert(0, {'$match': {'status': 'OK'}})
if tag:
pl.insert(0, {'$match': {'tag': tag}})
data = collection.aggregate(pl)
for rec in data:
if do_round:
x = int(round(rec['x']))
else:
x = rec['x']
column = chart_data[x]
column['x'] = x
for k in y_keys:
column[k] = column.get(k) or rec.get(k)
lines = collections.defaultdict(list) lines = collections.defaultdict(list)
@ -61,20 +82,17 @@ def generate_chart(chart_str, db, doc_folder, tag):
* *
''' % dict(title=title) ''' % dict(title=title)
axes_keys = sorted(axes.keys())
table += ''.join((' - %s\n' % axes[k]) for k in axes_keys) table += ''.join((' - %s\n' % axes[k]) for k in axes_keys)
y_keys = set(axes.keys()) ^ set('x') for _, chart_rec in sorted(chart_data.items(), key=lambda a: a[0]):
for chart_rec in chart_data:
for k in y_keys: for k in y_keys:
lines[k].append((chart_rec['x'], chart_rec[k])) if chart_rec[k]:
lines[k].append((chart_rec['x'], chart_rec[k]))
values = [] values = []
for v in axes_keys: for v in axes_keys:
cv = chart_rec[v] or 0 cv = '.' if chart_rec[v] is None else chart_rec[v]
patt = ' - %%%s' % ('d' if isinstance(cv, int) else '.1f') patt = ' - %%%s' % ('.1f' if isinstance(cv, float) else 's')
values.append(patt % cv) values.append(patt % cv)
table += (' *\n' + table += (' *\n' +
@ -96,8 +114,12 @@ def generate_chart(chart_str, db, doc_folder, tag):
abs_chart_filename = '%s.svg' % os.path.join(doc_folder, chart_filename) abs_chart_filename = '%s.svg' % os.path.join(doc_folder, chart_filename)
xy_chart.render_to_file(abs_chart_filename) xy_chart.render_to_file(abs_chart_filename)
doc = '.. image:: %s.*\n\n' % chart_filename doc = ''
doc += table if show_chart:
doc += '.. image:: %s.*\n\n' % chart_filename
if show_table:
doc += table
return doc return doc
@ -172,11 +194,23 @@ def generate_report(scenario, base_dir, mongo_url, db_name, doc_folder,
_make_dir(doc_folder) _make_dir(doc_folder)
jinja_env = jinja2.Environment() jinja_env = jinja2.Environment()
jinja_env.filters['chart'] = functools.partial( jinja_env.filters['chart_and_table'] = functools.partial(
generate_chart, generate_chart,
db=db, db=db,
doc_folder=doc_folder, doc_folder=doc_folder,
tag=tag) tag=tag)
jinja_env.filters['chart'] = functools.partial(
generate_chart,
db=db,
doc_folder=doc_folder,
tag=tag,
show_table=False)
jinja_env.filters['table'] = functools.partial(
generate_chart,
db=db,
doc_folder=doc_folder,
tag=tag,
show_chart=False)
jinja_env.filters['info'] = functools.partial( jinja_env.filters['info'] = functools.partial(
generate_info, generate_info,
db=db, db=db,

View File

@ -1,7 +1,6 @@
#!/usr/bin/python #!/usr/bin/python
import copy import copy
import os import os
import random
import signal import signal
import tempfile import tempfile
@ -147,8 +146,8 @@ def run(module):
client_summary = client_data['summary']['client'] client_summary = client_data['summary']['client']
record = dict(start=client_summary['start'], record = dict(start=client_summary['start'],
end=client_summary['end'], end=client_summary['end'],
client=client_summary) client=client_summary)
if 'round_trip' in client_data['summary']: if 'round_trip' in client_data['summary']:
round_trip_summary = client_data['summary']['round_trip'] round_trip_summary = client_data['summary']['round_trip']

View File

@ -5,8 +5,70 @@ This report is result of `message_queue_performance`_ execution
with `Oslo.messaging Simulator`_ with `Oslo.messaging Simulator`_
Test Case 1: RPC CALL Throughput Test RPC CALL fail-over throughput test
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
**Summary**
{{'''
title: Execution summary
fields:
a1: Client sent, msg
b1: Server received, msg
b2: Client received replies, msg
b3: Loss, msg
c1: Avg. request latency, ms
c2: Max request latency, ms
c3: Avg. round-trip latency, ms
c4: Max round-trip latency, ms
collection: records
pipeline:
- $match: { task: omsimulator, mode: call }
- $project:
a1: "$client.count"
b1: "$server.count"
b2: "$round_trip.count"
b3: { $subtract: ["$client.count", "$round_trip.count" ] }
c1: { $multiply: ["$server.latency", 1000] }
c2: { $multiply: ["$server.max_latency", 1000] }
c3: { $multiply: ["$round_trip.latency", 1000] }
c4: { $multiply: ["$round_trip.max_latency", 1000] }
''' | info
}}
**Message flow**
{{'''
title: RPC CALL message flow
axes:
x: time
y1: sent, msg
y2: received, msg
y3: round-trip, msg
y4: latency, ms
chart: line
collection: series
pipelines:
-
- $match: { task: omsimulator, mode: call, name: client_0 }
- $project:
x: "$timestamp"
y1: "$count"
-
- $match: { task: omsimulator, mode: call, name: server }
- $project:
x: "$timestamp"
y2: "$count"
-
- $match: { task: omsimulator, mode: call, name: round_trip_0 }
- $project:
x: "$timestamp"
y3: "$count"
y4: { $multiply: ["$latency", 1000] }
''' | chart
}}
**Messages sent by the client** **Messages sent by the client**
@ -18,15 +80,33 @@ Test Case 1: RPC CALL Throughput Test
chart: line chart: line
collection: series collection: series
pipeline: pipeline:
- { $match: { task: omsimulator, mode: call, name: client_0 }} - $match: { task: omsimulator, mode: call, name: client_0 }
- { $project: { x: "$seq", - $project:
y: "$count" x: "$seq"
}} y: "$count"
- { $sort: { x: 1 }} ''' | chart_and_table
''' | chart
}} }}
**Replies received by the client** **Messages received by the server**
{{'''
title: RPC CALL received messages
axes:
x: time
y: sent, msg
y2: latency, ms
chart: line
collection: series
pipeline:
- $match: { task: omsimulator, mode: call, name: server }
- $project:
x: "$seq"
y: "$count"
y2: { $multiply: ["$latency", 1000] }
''' | chart_and_table
}}
**Round-trip messages received by the client**
{{''' {{'''
title: RPC CALL round-trip messages title: RPC CALL round-trip messages
@ -37,11 +117,10 @@ Test Case 1: RPC CALL Throughput Test
chart: line chart: line
collection: series collection: series
pipeline: pipeline:
- { $match: { task: omsimulator, mode: call, name: round_trip_0 }} - $match: { task: omsimulator, mode: call, name: round_trip_0 }
- { $project: { x: "$seq", - $project:
y: "$count", x: "$seq"
y2: { $multiply: ["$latency", 1000] } y: "$count"
}} y2: { $multiply: ["$latency", 1000] }
- { $sort: { x: 1 }} ''' | chart_and_table
''' | chart
}} }}

View File

@ -36,7 +36,7 @@ execution:
tasks: tasks:
- omsimulator: - omsimulator:
mode: call mode: call
duration: 100 duration: 50
threads: 1 threads: 1
host_count: 1 host_count: 1
url: {{ rabbit_url }} url: {{ rabbit_url }}

View File

@ -5,8 +5,8 @@ This report is result of `message_queue_performance`_ execution
with `Oslo.messaging Simulator`_ with `Oslo.messaging Simulator`_
RPC CAST failover throughput test RPC CAST fail-over throughput test
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
**Summary** **Summary**
@ -30,6 +30,32 @@ RPC CAST failover throughput test
''' | info ''' | info
}} }}
**Message flow**
{{'''
title: RPC CAST message flow
axes:
x: time
y1: sent, msg
y2: received, msg
y3: latency, ms
chart: line
collection: series
pipelines:
-
- $match: { task: omsimulator, mode: cast, name: client_0 }
- $project:
x: "$timestamp"
y1: "$count"
-
- $match: { task: omsimulator, mode: cast, name: server }
- $project:
x: "$timestamp"
y2: "$count"
y3: { $multiply: ["$latency", 1000] }
''' | chart
}}
**Messages sent by the client** **Messages sent by the client**
@ -41,12 +67,11 @@ RPC CAST failover throughput test
chart: line chart: line
collection: series collection: series
pipeline: pipeline:
- { $match: { task: omsimulator, mode: cast, name: client_0 }} - $match: { task: omsimulator, mode: cast, name: client_0 }
- { $project: { x: "$seq", - $project:
y: "$count" x: "$seq"
}} y: "$count"
- { $sort: { x: 1 }} ''' | chart_and_table
''' | chart
}} }}
**Messages received by the server** **Messages received by the server**
@ -60,11 +85,10 @@ RPC CAST failover throughput test
chart: line chart: line
collection: series collection: series
pipeline: pipeline:
- { $match: { task: omsimulator, mode: cast, name: server }} - $match: { task: omsimulator, mode: cast, name: server }
- { $project: { x: "$seq", - $project:
y: "$count", x: "$seq",
y2: { $multiply: ["$latency", 1000] } y: "$count"
}} y2: { $multiply: ["$latency", 1000] }
- { $sort: { x: 1 }} ''' | chart_and_table
''' | chart
}} }}

View File

@ -36,7 +36,7 @@ execution:
tasks: tasks:
- omsimulator: - omsimulator:
mode: cast mode: cast
duration: 100 duration: 50
threads: 1 threads: 1
host_count: 1 host_count: 1
url: {{ rabbit_url }} url: {{ rabbit_url }}

View File

@ -38,7 +38,7 @@ received by the client. Also the number of lost messages is calculated.
y4: "$lost" y4: "$lost"
}} }}
- { $sort: { x: 1 }} - { $sort: { x: 1 }}
''' | chart ''' | chart_and_table
}} }}
@ -74,7 +74,7 @@ depending on number of concurrent threads.
y5: { $multiply: [ "$rabbit_total", 100 ] } y5: { $multiply: [ "$rabbit_total", 100 ] }
}} }}
- { $sort: { x: 1 }} - { $sort: { x: 1 }}
''' | chart ''' | chart_and_table
}} }}
@ -108,7 +108,7 @@ messages is calculated.
y3: "$lost" y3: "$lost"
}} }}
- { $sort: { x: 1 }} - { $sort: { x: 1 }}
''' | chart ''' | chart_and_table
}} }}
@ -141,7 +141,7 @@ depending on number of concurrent threads.
y4: { $multiply: [ "$rabbit_total", 100 ] } y4: { $multiply: [ "$rabbit_total", 100 ] }
}} }}
- { $sort: { x: 1 }} - { $sort: { x: 1 }}
''' | chart ''' | chart_and_table
}} }}
@ -175,7 +175,7 @@ messages is calculated.
y3: "$lost" y3: "$lost"
}} }}
- { $sort: { x: 1 }} - { $sort: { x: 1 }}
''' | chart ''' | chart_and_table
}} }}
@ -208,7 +208,7 @@ depending on number of concurrent threads.
y4: { $multiply: [ "$rabbit_total", 100 ] } y4: { $multiply: [ "$rabbit_total", 100 ] }
}} }}
- { $sort: { x: 1 }} - { $sort: { x: 1 }}
''' | chart ''' | chart_and_table
}} }}
.. references: .. references: