Browse Source

Multicharts and failover reports

Ilya Shakhat 3 years ago
parent
commit
c33edfecb5

+ 51
- 17
performa/engine/report.py View File

@@ -33,24 +33,45 @@ from performa.engine import utils
33 33
 LOG = logging.getLogger(__name__)
34 34
 
35 35
 
36
-def generate_chart(chart_str, db, doc_folder, tag):
36
+def generate_chart(chart_str, db, doc_folder, tag,
37
+                   show_chart=True, show_table=True):
37 38
     chart = yaml.safe_load(chart_str)
38
-    pipeline = chart.get('pipeline')
39
+    pipeline = chart.get('pipeline')  # single pipeline
40
+    pipelines = chart.get('pipelines')  # multiple pipelines
39 41
     title = chart.get('title')
40 42
     fill = chart.get('fill') or False
41 43
     axes = chart.get('axes') or dict(x='x', y='y')
44
+    do_round = show_table
42 45
 
43 46
     collection_name = chart.get('collection') or 'records'
44 47
     collection = db.get_collection(collection_name)
45 48
 
46 49
     LOG.debug('Title: %s', title)
47 50
 
48
-    pipeline.insert(0, {'$match': {'status': 'OK'}})
51
+    axes_keys = sorted(axes.keys())
52
+    y_keys = set(axes.keys()) ^ set('x')
49 53
 
50
-    if tag:
51
-        pipeline.insert(0, {'$match': {'tag': tag}})
54
+    chart_data = collections.defaultdict(dict)
55
+
56
+    for pl in (pipelines or [pipeline]):
57
+        pl.insert(0, {'$match': {'status': 'OK'}})
52 58
 
53
-    chart_data = collection.aggregate(pipeline)
59
+        if tag:
60
+            pl.insert(0, {'$match': {'tag': tag}})
61
+
62
+        data = collection.aggregate(pl)
63
+
64
+        for rec in data:
65
+            if do_round:
66
+                x = int(round(rec['x']))
67
+            else:
68
+                x = rec['x']
69
+
70
+            column = chart_data[x]
71
+            column['x'] = x
72
+
73
+            for k in y_keys:
74
+                column[k] = column.get(k) or rec.get(k)
54 75
 
55 76
     lines = collections.defaultdict(list)
56 77
 
@@ -61,20 +82,17 @@ def generate_chart(chart_str, db, doc_folder, tag):
61 82
    *
62 83
 ''' % dict(title=title)
63 84
 
64
-    axes_keys = sorted(axes.keys())
65
-
66 85
     table += ''.join(('     - %s\n' % axes[k]) for k in axes_keys)
67 86
 
68
-    y_keys = set(axes.keys()) ^ set('x')
69
-
70
-    for chart_rec in chart_data:
87
+    for _, chart_rec in sorted(chart_data.items(), key=lambda a: a[0]):
71 88
         for k in y_keys:
72
-            lines[k].append((chart_rec['x'], chart_rec[k]))
89
+            if chart_rec[k]:
90
+                lines[k].append((chart_rec['x'], chart_rec[k]))
73 91
 
74 92
         values = []
75 93
         for v in axes_keys:
76
-            cv = chart_rec[v] or 0
77
-            patt = '     - %%%s' % ('d' if isinstance(cv, int) else '.1f')
94
+            cv = '.' if chart_rec[v] is None else chart_rec[v]
95
+            patt = '     - %%%s' % ('.1f' if isinstance(cv, float) else 's')
78 96
             values.append(patt % cv)
79 97
 
80 98
         table += ('   *\n' +
@@ -96,8 +114,12 @@ def generate_chart(chart_str, db, doc_folder, tag):
96 114
     abs_chart_filename = '%s.svg' % os.path.join(doc_folder, chart_filename)
97 115
     xy_chart.render_to_file(abs_chart_filename)
98 116
 
99
-    doc = '.. image:: %s.*\n\n' % chart_filename
100
-    doc += table
117
+    doc = ''
118
+    if show_chart:
119
+        doc += '.. image:: %s.*\n\n' % chart_filename
120
+
121
+    if show_table:
122
+        doc += table
101 123
 
102 124
     return doc
103 125
 
@@ -172,11 +194,23 @@ def generate_report(scenario, base_dir, mongo_url, db_name, doc_folder,
172 194
     _make_dir(doc_folder)
173 195
 
174 196
     jinja_env = jinja2.Environment()
175
-    jinja_env.filters['chart'] = functools.partial(
197
+    jinja_env.filters['chart_and_table'] = functools.partial(
176 198
         generate_chart,
177 199
         db=db,
178 200
         doc_folder=doc_folder,
179 201
         tag=tag)
202
+    jinja_env.filters['chart'] = functools.partial(
203
+        generate_chart,
204
+        db=db,
205
+        doc_folder=doc_folder,
206
+        tag=tag,
207
+        show_table=False)
208
+    jinja_env.filters['table'] = functools.partial(
209
+        generate_chart,
210
+        db=db,
211
+        doc_folder=doc_folder,
212
+        tag=tag,
213
+        show_chart=False)
180 214
     jinja_env.filters['info'] = functools.partial(
181 215
         generate_info,
182 216
         db=db,

+ 2
- 3
performa/modules/omsimulator.py View File

@@ -1,7 +1,6 @@
1 1
 #!/usr/bin/python
2 2
 import copy
3 3
 import os
4
-import random
5 4
 import signal
6 5
 import tempfile
7 6
 
@@ -147,8 +146,8 @@ def run(module):
147 146
         client_summary = client_data['summary']['client']
148 147
 
149 148
         record = dict(start=client_summary['start'],
150
-                       end=client_summary['end'],
151
-                       client=client_summary)
149
+                      end=client_summary['end'],
150
+                      client=client_summary)
152 151
 
153 152
         if 'round_trip' in client_data['summary']:
154 153
             round_trip_summary = client_data['summary']['round_trip']

+ 95
- 16
performa/scenarios/mq/omsimulator-ha-call.rst View File

@@ -5,8 +5,70 @@ This report is result of `message_queue_performance`_ execution
5 5
 with `Oslo.messaging Simulator`_
6 6
 
7 7
 
8
-Test Case 1: RPC CALL Throughput Test
9
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
8
+RPC CALL fail-over throughput test
9
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
10
+
11
+**Summary**
12
+
13
+{{'''
14
+    title: Execution summary
15
+    fields:
16
+      a1: Client sent, msg
17
+      b1: Server received, msg
18
+      b2: Client received replies, msg
19
+      b3: Loss, msg
20
+      c1: Avg. request latency, ms
21
+      c2: Max request latency, ms
22
+      c3: Avg. round-trip latency, ms
23
+      c4: Max round-trip latency, ms
24
+    collection: records
25
+    pipeline:
26
+    - $match: { task: omsimulator, mode: call }
27
+    - $project:
28
+        a1: "$client.count"
29
+        b1: "$server.count"
30
+        b2: "$round_trip.count"
31
+        b3: { $subtract: ["$client.count", "$round_trip.count" ] }
32
+        c1: { $multiply: ["$server.latency", 1000] }
33
+        c2: { $multiply: ["$server.max_latency", 1000] }
34
+        c3: { $multiply: ["$round_trip.latency", 1000] }
35
+        c4: { $multiply: ["$round_trip.max_latency", 1000] }
36
+''' | info
37
+}}
38
+
39
+
40
+**Message flow**
41
+
42
+{{'''
43
+    title: RPC CALL message flow
44
+    axes:
45
+      x: time
46
+      y1: sent, msg
47
+      y2: received, msg
48
+      y3: round-trip, msg
49
+      y4: latency, ms
50
+    chart: line
51
+    collection: series
52
+    pipelines:
53
+    -
54
+      - $match: { task: omsimulator, mode: call, name: client_0 }
55
+      - $project:
56
+          x: "$timestamp"
57
+          y1: "$count"
58
+    -
59
+      - $match: { task: omsimulator, mode: call, name: server }
60
+      - $project:
61
+          x: "$timestamp"
62
+          y2: "$count"
63
+    -
64
+      - $match: { task: omsimulator, mode: call, name: round_trip_0 }
65
+      - $project:
66
+          x: "$timestamp"
67
+          y3: "$count"
68
+          y4: { $multiply: ["$latency", 1000] }
69
+''' | chart
70
+}}
71
+
10 72
 
11 73
 **Messages sent by the client**
12 74
 
@@ -18,15 +80,33 @@ Test Case 1: RPC CALL Throughput Test
18 80
     chart: line
19 81
     collection: series
20 82
     pipeline:
21
-    - { $match: { task: omsimulator, mode: call, name: client_0 }}
22
-    - { $project: { x: "$seq",
23
-                    y: "$count"
24
-                  }}
25
-    - { $sort: { x: 1 }}
26
-''' | chart
83
+    - $match: { task: omsimulator, mode: call, name: client_0 }
84
+    - $project:
85
+        x: "$seq"
86
+        y: "$count"
87
+''' | chart_and_table
88
+}}
89
+
90
+**Messages received by the server**
91
+
92
+{{'''
93
+    title: RPC CALL received messages
94
+    axes:
95
+      x: time
96
+      y: sent, msg
97
+      y2: latency, ms
98
+    chart: line
99
+    collection: series
100
+    pipeline:
101
+    - $match: { task: omsimulator, mode: call, name: server }
102
+    - $project:
103
+        x: "$seq"
104
+        y: "$count"
105
+        y2: { $multiply: ["$latency", 1000] }
106
+''' | chart_and_table
27 107
 }}
28 108
 
29
-**Replies received by the client**
109
+**Round-trip messages received by the client**
30 110
 
31 111
 {{'''
32 112
     title: RPC CALL round-trip messages
@@ -37,11 +117,10 @@ Test Case 1: RPC CALL Throughput Test
37 117
     chart: line
38 118
     collection: series
39 119
     pipeline:
40
-    - { $match: { task: omsimulator, mode: call, name: round_trip_0 }}
41
-    - { $project: { x: "$seq",
42
-                    y: "$count",
43
-                    y2: { $multiply: ["$latency", 1000] }
44
-                  }}
45
-    - { $sort: { x: 1 }}
46
-''' | chart
120
+    - $match: { task: omsimulator, mode: call, name: round_trip_0 }
121
+    - $project:
122
+        x: "$seq"
123
+        y: "$count"
124
+        y2: { $multiply: ["$latency", 1000] }
125
+''' | chart_and_table
47 126
 }}

+ 1
- 1
performa/scenarios/mq/omsimulator-ha-call.yaml View File

@@ -36,7 +36,7 @@ execution:
36 36
     tasks:
37 37
     - omsimulator:
38 38
         mode: call
39
-        duration: 100
39
+        duration: 50
40 40
         threads: 1
41 41
         host_count: 1
42 42
         url: {{ rabbit_url }}

+ 39
- 15
performa/scenarios/mq/omsimulator-ha-cast.rst View File

@@ -5,8 +5,8 @@ This report is result of `message_queue_performance`_ execution
5 5
 with `Oslo.messaging Simulator`_
6 6
 
7 7
 
8
-RPC CAST failover throughput test
9
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
8
+RPC CAST fail-over throughput test
9
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
10 10
 
11 11
 **Summary**
12 12
 
@@ -30,6 +30,32 @@ RPC CAST failover throughput test
30 30
 ''' | info
31 31
 }}
32 32
 
33
+**Message flow**
34
+
35
+{{'''
36
+    title: RPC CAST message flow
37
+    axes:
38
+      x: time
39
+      y1: sent, msg
40
+      y2: received, msg
41
+      y3: latency, ms
42
+    chart: line
43
+    collection: series
44
+    pipelines:
45
+    -
46
+      - $match: { task: omsimulator, mode: cast, name: client_0 }
47
+      - $project:
48
+          x: "$timestamp"
49
+          y1: "$count"
50
+    -
51
+      - $match: { task: omsimulator, mode: cast, name: server }
52
+      - $project:
53
+          x: "$timestamp"
54
+          y2: "$count"
55
+          y3: { $multiply: ["$latency", 1000] }
56
+''' | chart
57
+}}
58
+
33 59
 
34 60
 **Messages sent by the client**
35 61
 
@@ -41,12 +67,11 @@ RPC CAST failover throughput test
41 67
     chart: line
42 68
     collection: series
43 69
     pipeline:
44
-    - { $match: { task: omsimulator, mode: cast, name: client_0 }}
45
-    - { $project: { x: "$seq",
46
-                    y: "$count"
47
-                  }}
48
-    - { $sort: { x: 1 }}
49
-''' | chart
70
+    - $match: { task: omsimulator, mode: cast, name: client_0 }
71
+    - $project:
72
+        x: "$seq"
73
+        y: "$count"
74
+''' | chart_and_table
50 75
 }}
51 76
 
52 77
 **Messages received by the server**
@@ -60,11 +85,10 @@ RPC CAST failover throughput test
60 85
     chart: line
61 86
     collection: series
62 87
     pipeline:
63
-    - { $match: { task: omsimulator, mode: cast, name: server }}
64
-    - { $project: { x: "$seq",
65
-                    y: "$count",
66
-                    y2: { $multiply: ["$latency", 1000] }
67
-                  }}
68
-    - { $sort: { x: 1 }}
69
-''' | chart
88
+    - $match: { task: omsimulator, mode: cast, name: server }
89
+    - $project:
90
+        x: "$seq",
91
+        y: "$count"
92
+        y2: { $multiply: ["$latency", 1000] }
93
+''' | chart_and_table
70 94
 }}

+ 1
- 1
performa/scenarios/mq/omsimulator-ha-cast.yaml View File

@@ -36,7 +36,7 @@ execution:
36 36
     tasks:
37 37
     - omsimulator:
38 38
         mode: cast
39
-        duration: 100
39
+        duration: 50
40 40
         threads: 1
41 41
         host_count: 1
42 42
         url: {{ rabbit_url }}

+ 6
- 6
performa/scenarios/mq/omsimulator.rst View File

@@ -38,7 +38,7 @@ received by the client. Also the number of lost messages is calculated.
38 38
                     y4: "$lost"
39 39
                   }}
40 40
     - { $sort: { x: 1 }}
41
-''' | chart
41
+''' | chart_and_table
42 42
 }}
43 43
 
44 44
 
@@ -74,7 +74,7 @@ depending on number of concurrent threads.
74 74
                     y5: { $multiply: [ "$rabbit_total", 100 ] }
75 75
                   }}
76 76
     - { $sort: { x: 1 }}
77
-''' | chart
77
+''' | chart_and_table
78 78
 }}
79 79
 
80 80
 
@@ -108,7 +108,7 @@ messages is calculated.
108 108
                     y3: "$lost"
109 109
                   }}
110 110
     - { $sort: { x: 1 }}
111
-''' | chart
111
+''' | chart_and_table
112 112
 }}
113 113
 
114 114
 
@@ -141,7 +141,7 @@ depending on number of concurrent threads.
141 141
                     y4: { $multiply: [ "$rabbit_total", 100 ] }
142 142
                   }}
143 143
     - { $sort: { x: 1 }}
144
-''' | chart
144
+''' | chart_and_table
145 145
 }}
146 146
 
147 147
 
@@ -175,7 +175,7 @@ messages is calculated.
175 175
                     y3: "$lost"
176 176
                   }}
177 177
     - { $sort: { x: 1 }}
178
-''' | chart
178
+''' | chart_and_table
179 179
 }}
180 180
 
181 181
 
@@ -208,7 +208,7 @@ depending on number of concurrent threads.
208 208
                     y4: { $multiply: [ "$rabbit_total", 100 ] }
209 209
                   }}
210 210
     - { $sort: { x: 1 }}
211
-''' | chart
211
+''' | chart_and_table
212 212
 }}
213 213
 
214 214
 .. references:

Loading…
Cancel
Save