Browse Source

Filter atop series inside the module

Introduce ability to filter series inside atop module.
The feature is configured via parameter 'filter', which
is a dictinary where keys as param names and values play role
of filter.

Change-Id: I553558a63879a3e6f66522a040b74b91e4884800
Ilya Shakhat 3 years ago
parent
commit
6e684cd841

+ 2
- 0
performa/engine/player.py View File

@@ -87,6 +87,8 @@ def play_execution(runner, execution_playbook):
87 87
                             rec.update(common)
88 88
                             series.append(rec)
89 89
                             LOG.debug('New time series: %s', rec)
90
+                else:
91
+                    LOG.warning('Play failed: %s', command_result)
90 92
 
91 93
     return records, series
92 94
 

+ 44
- 8
performa/modules/atop.py View File

@@ -1,5 +1,6 @@
1 1
 #!/usr/bin/python
2 2
 
3
+import functools
3 4
 import os
4 5
 import re
5 6
 import tempfile
@@ -145,10 +146,7 @@ def normalize_point(point):
145 146
     return point
146 147
 
147 148
 
148
-def parse_output(raw, filter_labels):
149
-    filter_labels = set(filter_labels)
150
-    series = []
151
-
149
+def parse_output(raw):
152 150
     active = False
153 151
     for line in raw.split('\n'):
154 152
         if line == 'SEP':
@@ -162,11 +160,44 @@ def parse_output(raw, filter_labels):
162 160
             m = re.match(pattern, line)
163 161
             if m:
164 162
                 point = m.groupdict()
165
-                if point['label'] in filter_labels:
166
-                    series.append(normalize_point(point))
163
+                yield normalize_point(point)
167 164
                 break
168 165
 
169
-    return series
166
+
167
+def make_filter_funcs(filters):
168
+    def _in_list(point, name, lst):
169
+        return point.get(name) in lst
170
+
171
+    def _match(point, name, patt):
172
+        return re.match(patt, point.get(name, ''))
173
+
174
+    funcs = []
175
+    for name, values in filters.items():
176
+        fn = None
177
+        if isinstance(values, list):
178
+            fn = functools.partial(_in_list, name=name, lst=set(values))
179
+        elif isinstance(values, str):
180
+            fn = functools.partial(_match, name=name, patt=re.compile(values))
181
+        if fn:
182
+            funcs.append(fn)
183
+    return funcs
184
+
185
+
186
+def run_filter_funcs(points, funcs):
187
+    for point in points:
188
+        accepted = True
189
+        for fn in funcs:
190
+            if not fn(point):
191
+                accepted = False
192
+                break
193
+
194
+        if accepted:
195
+            yield point
196
+
197
+
198
+def parse(raw, filters):
199
+    funcs = make_filter_funcs(filters)
200
+    return list(run_filter_funcs(parse_output(raw), funcs))
170 201
 
171 202
 
172 203
 def start(module):
@@ -206,13 +237,17 @@ def stop(module):
206 237
 
207 238
     # grab data
208 239
     labels = module.params['labels'] or ALL_LABELS
240
+    ft = module.params.get('filter')
241
+    if ft and 'label' in ft:
242
+        labels = ft['label']
243
+
209 244
     cmd = ('atop -r %(file)s -P %(labels)s' %
210 245
            dict(file=ATOP_FILE_NAME, labels=','.join(labels)))
211 246
 
212 247
     rc, stdout, stderr = module.run_command(cmd)
213 248
 
214 249
     try:
215
-        series = parse_output(stdout, labels)
250
+        series = parse(stdout, module.params.get('filter', {}))
216 251
         module.exit_json(series=series)
217 252
     except Exception as e:
218 253
         module.fail_json(msg=str(e), stderr=stderr, rc=rc)
@@ -224,6 +259,7 @@ def main():
224 259
             command=dict(required=True, choices=['start', 'stop']),
225 260
             interval=dict(type='int', default=1),
226 261
             labels=dict(type='list'),
262
+            filter=dict(type='dict', default={}),
227 263
         ))
228 264
 
229 265
     command = module.params['command']

+ 4
- 2
performa/scenarios/db/sysbench.yaml View File

@@ -35,7 +35,7 @@ execution:
35 35
       threads: [ 5, 10, 15, 20, 30, 40, 50 ]
36 36
     tasks:
37 37
     - sysbench_oltp:
38
-        duration: 60
38
+        duration: 10
39 39
         mysql_host: {{ mysql_endpoint }}
40 40
         mysql_port: {{ mysql_port }}
41 41
         mysql_db: sbtest
@@ -44,7 +44,9 @@ execution:
44 44
     tasks:
45 45
     - atop:
46 46
         command: stop
47
-        labels: [ PRC ]
47
+        filter:
48
+          name: mysqld
49
+          label: [ PRC ]
48 50
 
49 51
 aggregation:
50 52
   -

+ 3
- 1
performa/scenarios/mq/rabbitmq.yaml View File

@@ -87,7 +87,9 @@ execution:
87 87
     tasks:
88 88
     - atop:
89 89
         command: stop
90
-        labels: [ PRC ]
90
+        filter:
91
+          name: beam.*
92
+          label: [ PRC ]
91 93
 
92 94
 aggregation:
93 95
   -

+ 35
- 6
performa/tests/test_atop.py View File

@@ -36,7 +36,8 @@ class TestAtop(testtools.TestCase):
36 36
                      'sys': 0.04, 'ticks_per_second': 100, 'time': '10:01:05',
37 37
                      'timestamp': 1456480865, 'user': 0.04, 'wait': 0.0}]
38 38
 
39
-        self.assertEqual(expected, atop.parse_output(_read_sample(), ['CPU']))
39
+        self.assertEqual(expected,
40
+                         atop.parse(_read_sample(), dict(label=['CPU'])))
40 41
 
41 42
     def test_parse_cpu(self):
42 43
         needle = {'cpu_id': 2, 'date': '2016/02/26', 'guest': 0.0,
@@ -45,7 +46,8 @@ class TestAtop(testtools.TestCase):
45 46
                   'sys': 0.03, 'ticks_per_second': 100, 'time': '10:01:05',
46 47
                   'timestamp': 1456480865, 'user': 0.03, 'wait': 0.0}
47 48
 
48
-        self.assertIn(needle, atop.parse_output(_read_sample(), ['cpu']))
49
+        self.assertIn(needle,
50
+                      atop.parse(_read_sample(), dict(label=['cpu'])))
49 51
 
50 52
     def test_parse_mem(self):
51 53
         expected = [
@@ -58,7 +60,8 @@ class TestAtop(testtools.TestCase):
58 60
              'label': 'MEM', 'page_size': 4096, 'phys': 8373075968,
59 61
              'slab': 298115072, 'time': '10:01:05', 'timestamp': 1456480865}]
60 62
 
61
-        self.assertEqual(expected, atop.parse_output(_read_sample(), ['MEM']))
63
+        self.assertEqual(expected,
64
+                         atop.parse(_read_sample(), dict(label=['MEM'])))
62 65
 
63 66
     def test_parse_net(self):
64 67
         needle = {'date': '2016/02/26', 'host': 'host', 'interval': 1,
@@ -66,7 +69,8 @@ class TestAtop(testtools.TestCase):
66 69
                   'label': 'NET', 'tcp_rx': 0, 'tcp_tx': 0, 'time': '10:01:04',
67 70
                   'timestamp': 1456480864, 'udp_rx': 0, 'udp_tx': 0}
68 71
 
69
-        self.assertIn(needle, atop.parse_output(_read_sample(), ['NET']))
72
+        self.assertIn(needle,
73
+                      atop.parse(_read_sample(), dict(label=['NET'])))
70 74
 
71 75
     def test_parse_prc(self):
72 76
         needle = {'current_cpu': 2, 'date': '2016/02/26', 'host': 'host',
@@ -76,7 +80,8 @@ class TestAtop(testtools.TestCase):
76 80
                   'sys': 0.02, 'ticks_per_second': 100, 'time': '10:01:04',
77 81
                   'timestamp': 1456480864, 'user': 0.01}
78 82
 
79
-        self.assertIn(needle, atop.parse_output(_read_sample(), ['PRC']))
83
+        self.assertIn(needle,
84
+                      atop.parse(_read_sample(), dict(label=['PRC'])))
80 85
 
81 86
     def test_parse_prm(self):
82 87
         needle = {'date': '2016/02/26', 'host': 'host', 'interval': 1,
@@ -87,4 +92,28 @@ class TestAtop(testtools.TestCase):
87 92
                   'timestamp': 1456480865, 'virtual': 17412096,
88 93
                   'virtual_growth': 0}
89 94
 
90
-        self.assertIn(needle, atop.parse_output(_read_sample(), ['PRM']))
95
+        self.assertIn(needle,
96
+                      atop.parse(_read_sample(), dict(label=['PRM'])))
97
+
98
+    def test_parse_match_name_regex(self):
99
+        expected = [{'current_cpu': 2, 'date': '2016/02/26', 'host': 'host',
100
+                     'interval': 1, 'label': 'PRC', 'name': 'dstat', 'nice': 0,
101
+                     'pid': 11014, 'priority': 120, 'realtime_priority': 0,
102
+                     'scheduling_policy': 0, 'sleep_avg': 0, 'state': 'S',
103
+                     'sys': 0.02, 'ticks_per_second': 100, 'time': '10:01:04',
104
+                     'timestamp': 1456480864, 'user': 0.01},
105
+                    {'current_cpu': 2, 'date': '2016/02/26', 'host': 'host',
106
+                     'interval': 1, 'label': 'PRC', 'name': 'dstat', 'nice': 0,
107
+                     'pid': 11014, 'priority': 120, 'realtime_priority': 0,
108
+                     'scheduling_policy': 0, 'sleep_avg': 0, 'state': 'S',
109
+                     'sys': 0.0, 'ticks_per_second': 100, 'time': '10:01:05',
110
+                     'timestamp': 1456480865, 'user': 0.02}]
111
+
112
+        filter = {
113
+            'name': 'dstat',
114
+            'label': ['PRC'],
115
+        }
116
+        self.assertEqual(expected, atop.parse(_read_sample(), filter))
117
+
118
+    def test_parse_no_filter(self):
119
+        self.assertEqual(43, len(atop.parse(_read_sample(), {})))

Loading…
Cancel
Save