Browse Source

Add atop module to collect system metrics

Ilya Shakhat 3 years ago
parent
commit
9a605c2a68

+ 1
- 0
performa/engine/ansible_runner.py View File

@@ -118,6 +118,7 @@ def _run(play_source, host_list):
118 118
         if tqm is not None:
119 119
             tqm.cleanup()
120 120
 
121
+    LOG.debug('Execution result: %s', storage)
121 122
     return storage
122 123
 
123 124
 

+ 2
- 1
performa/engine/player.py View File

@@ -75,7 +75,8 @@ def play_execution(execution_playbook):
75 75
                         if isinstance(v, list) or isinstance(v, dict):
76 76
                             del record[k]
77 77
 
78
-                    del record['stdout']
78
+                    if 'stdout' in record:
79
+                        del record['stdout']
79 80
 
80 81
                     LOG.debug('Record: %s', record)
81 82
                     records.append(record)

+ 240
- 0
performa/modules/atop.py View File

@@ -0,0 +1,240 @@
1
+#!/usr/bin/python
2
+
3
+import re
4
+
5
+ATOP_FILE_NAME = '/tmp/performa.atop'
6
+UNIQUE_NAME = 'performa_atop'
7
+
8
+PREFIX_PATTERN = (
9
+    '(?P<host>\S+)\s+'
10
+    '(?P<timestamp>\d+)\s+'
11
+    '(?P<date>\S+)\s+'
12
+    '(?P<time>\S+)\s+'
13
+    '(?P<interval>\w+)\s+'
14
+)
15
+
16
+CPU_TOTAL_PATTERN = re.compile(
17
+    '(?P<label>CPU)\s+' +
18
+    PREFIX_PATTERN +
19
+    '(?P<ticks_per_second>\d+)\s+'
20
+    '(?P<cpu_count>\d+)\s+'
21
+    '(?P<sys_ticks>\d+)\s+'
22
+    '(?P<user_ticks>\d+)\s+'
23
+    '(?P<nice_ticks>\d+)\s+'
24
+    '(?P<idle_ticks>\d+)\s+'
25
+    '(?P<wait_ticks>\d+)\s+'
26
+    '(?P<irq_ticks>\d+)\s+'
27
+    '(?P<softirq_ticks>\d+)\s+'
28
+    '(?P<steal_ticks>\d+)\s+'
29
+    '(?P<guest_ticks>\d+)',
30
+)
31
+
32
+CPU_PATTERN = re.compile(
33
+    '(?P<label>cpu)\s+' +
34
+    PREFIX_PATTERN +
35
+    '(?P<ticks_per_second>\d+)\s+'
36
+    '(?P<cpu_id>\d+)\s+'
37
+    '(?P<sys_ticks>\d+)\s+'
38
+    '(?P<user_ticks>\d+)\s+'
39
+    '(?P<nice_ticks>\d+)\s+'
40
+    '(?P<idle_ticks>\d+)\s+'
41
+    '(?P<wait_ticks>\d+)\s+'
42
+    '(?P<irq_ticks>\d+)\s+'
43
+    '(?P<softirq_ticks>\d+)\s+'
44
+    '(?P<steal_ticks>\d+)\s+'
45
+    '(?P<guest_ticks>\d+)',
46
+)
47
+
48
+MEM_PATTERN = re.compile(
49
+    '(?P<label>MEM)\s+' +
50
+    PREFIX_PATTERN +
51
+    '(?P<page_size>\d+)\s+'
52
+    '(?P<phys_pages>\d+)\s+'
53
+    '(?P<free_pages>\d+)\s+'
54
+    '(?P<cache_pages>\d+)\s+'
55
+    '(?P<buffer_pages>\d+)\s+'
56
+    '(?P<slab_pages>\d+)\s+'
57
+    '(?P<dirty_pages>\d+)'
58
+)
59
+
60
+NET_UPPER_PATTERN = re.compile(
61
+    '(?P<label>NET)\s+' +
62
+    PREFIX_PATTERN +
63
+    'upper\s+'
64
+    '(?P<tcp_rx>\d+)\s+'
65
+    '(?P<tcp_tx>\d+)\s+'
66
+    '(?P<udp_rx>\d+)\s+'
67
+    '(?P<udp_tx>\d+)\s+'
68
+    '(?P<ip_rx>\d+)\s+'
69
+    '(?P<ip_tx>\d+)\s+'
70
+    '(?P<ip_dx>\d+)\s+'
71
+    '(?P<ip_fx>\d+)'
72
+)
73
+
74
+NET_PATTERN = re.compile(
75
+    '(?P<label>NET)\s+' +
76
+    PREFIX_PATTERN +
77
+    '(?P<interface>\S+)\s+'
78
+    '(?P<rx_pkt>\d+)\s+'
79
+    '(?P<tx_pkt>\d+)\s+'
80
+    '(?P<rx_bytes>\d+)\s+'
81
+    '(?P<tx_bytes>\d+)\s+'
82
+    '(?P<speed>\d+)\s+'
83
+    '(?P<duplex_command>\d+)'
84
+)
85
+
86
+PRC_PATTERN = re.compile(
87
+    '(?P<label>PRC)\s+' +
88
+    PREFIX_PATTERN +
89
+    '(?P<pid>\d+)\s+'
90
+    '\((?P<name>.+)\)\s+'
91
+    '(?P<state>\S+)\s+'
92
+    '(?P<ticks_per_second>\d+)\s+'
93
+    '(?P<user_ticks>\d+)\s+'
94
+    '(?P<sys_ticks>\d+)\s+'
95
+    '(?P<nice>\d+)\s+'
96
+    '(?P<priority>\d+)\s+'
97
+    '(?P<realtime_priority>\d+)\s+'
98
+    '(?P<scheduling_policy>\d+)\s+'
99
+    '(?P<current_cpu>\d+)\s+'
100
+    '(?P<sleep_avg>\d+)'
101
+)
102
+
103
+PRM_PATTERN = re.compile(
104
+    '(?P<label>PRM)\s+' +
105
+    PREFIX_PATTERN +
106
+    '(?P<pid>\d+)\s+'
107
+    '\((?P<name>.+)\)\s+'
108
+    '(?P<state>\S+)\s+'
109
+    '(?P<page_size>\d+)\s+'
110
+    '(?P<virtual_kb>\d+)\s+'
111
+    '(?P<resident_kb>\d+)\s+'
112
+    '(?P<shared_kb>\d+)\s+'
113
+    '(?P<virtual_growth_kb>\d+)\s+'
114
+    '(?P<resident_growth_kb>\d+)\s+'
115
+    '(?P<minor_page_faults>\d+)\s+'
116
+    '(?P<major_page_faults>\d+)'
117
+)
118
+
119
+PATTERNS = [CPU_TOTAL_PATTERN, CPU_PATTERN, MEM_PATTERN,
120
+            NET_UPPER_PATTERN, NET_PATTERN, PRC_PATTERN, PRM_PATTERN]
121
+
122
+ALL_LABELS = ['CPU', 'cpu', 'MEM', 'NET', 'PRC', 'PRM']
123
+
124
+
125
+def normalize_point(point):
126
+    # interpret strings into numbers
127
+    for k, v in point.items():
128
+        if v.isdigit():
129
+            point[k] = int(v)
130
+
131
+    # convert measurement units
132
+    for k, v in point.items():
133
+        if k[-6:] == '_pages':
134
+            point[k[:-6]] = v * point['page_size']
135
+            del point[k]
136
+        elif k[-6:] == '_ticks':
137
+            point[k[:-6]] = float(v) / point['ticks_per_second']
138
+            del point[k]
139
+        elif k[-3:] == '_kb':
140
+            point[k[:-3]] = v * 1024
141
+            del point[k]
142
+
143
+    return point
144
+
145
+
146
+def parse_output(raw, filter_labels):
147
+    filter_labels = set(filter_labels)
148
+    series = []
149
+
150
+    active = False
151
+    for line in raw.split('\n'):
152
+        if line == 'SEP':
153
+            active = True
154
+            continue
155
+
156
+        if not active:
157
+            continue
158
+
159
+        for pattern in PATTERNS:
160
+            m = re.match(pattern, line)
161
+            if m:
162
+                point = m.groupdict()
163
+                if point['label'] in filter_labels:
164
+                    series.append(normalize_point(point))
165
+                break
166
+
167
+    return series
168
+
169
+
170
+def start(module):
171
+    # clear the file
172
+    cmd = 'rm %s' % ATOP_FILE_NAME
173
+    module.run_command(cmd)
174
+
175
+    # start atop as daemon
176
+    cmd = ('daemon -n %(name)s -- atop -w %(file)s %(interval)s' %
177
+           dict(name=UNIQUE_NAME, file=ATOP_FILE_NAME,
178
+                interval=module.params['interval']))
179
+
180
+    rc, stdout, stderr = module.run_command(cmd)
181
+    result = dict(changed=True, rc=rc, stdout=stdout, stderr=stderr, cmd=cmd)
182
+
183
+    if rc:
184
+        module.fail_json(msg='Failed to start atop', **result)
185
+    else:
186
+        # sleep until file is created
187
+        for timeout in range(10):
188
+            if os.path.exists(ATOP_FILE_NAME):
189
+                break
190
+
191
+            module.run_command('sleep 1')
192
+
193
+        module.exit_json(**result)
194
+
195
+
196
+def stop(module):
197
+    # stop atop
198
+    cmd = 'daemon -n %(name)s --stop' % dict(name=UNIQUE_NAME)
199
+
200
+    rc, stdout, stderr = module.run_command(cmd)
201
+
202
+    if rc:
203
+        module.fail_json(msg='Failed to stop atop', rc=rc, stderr=stderr)
204
+
205
+    # grab data
206
+    labels = module.params['labels'] or ALL_LABELS
207
+    cmd = ('atop -r %(file)s -P %(labels)s' %
208
+           dict(file=ATOP_FILE_NAME, labels=','.join(labels)))
209
+
210
+    rc, stdout, stderr = module.run_command(cmd)
211
+
212
+    try:
213
+        series = parse_output(stdout, labels)
214
+        module.exit_json(series=series)
215
+    except Exception as e:
216
+        module.fail_json(msg=str(e), stderr=stderr, rc=rc)
217
+
218
+
219
+def main():
220
+    module = AnsibleModule(
221
+        argument_spec=dict(
222
+            command=dict(required=True, choices=['start', 'stop']),
223
+            interval=dict(type='int', default=1),
224
+            labels=dict(type='list'),
225
+        ))
226
+
227
+    command = module.params['command']
228
+
229
+    if command == 'start':
230
+        start(module)
231
+    elif command == 'stop':
232
+        stop(module)
233
+    else:
234
+        module.fail_json(msg='Unsupported command: %s' % command)
235
+
236
+
237
+from ansible.module_utils.basic import *  # noqa
238
+
239
+if __name__ == '__main__':
240
+    main()

+ 16
- 0
performa/scenarios/db/sysbench.yaml View File

@@ -12,6 +12,16 @@ setup:
12 12
       become: yes
13 13
       become_user: root
14 14
       become_method: sudo
15
+    - name: installing atop
16
+      apt:
17
+        name: atop, daemon
18
+      become: yes
19
+      become_user: root
20
+      become_method: sudo
21
+  -
22
+    hosts: $target
23
+    tasks:
24
+    - atop: command=start
15 25
 
16 26
 execution:
17 27
   -
@@ -21,6 +31,12 @@ execution:
21 31
     tasks:
22 32
     - sysbench_oltp:
23 33
         duration: 10
34
+  -
35
+    hosts: $target
36
+    tasks:
37
+    - atop:
38
+        command: stop
39
+        labels: [ CPU, PRC, PRM ]
24 40
 
25 41
 report:
26 42
   template: sysbench.rst

+ 50
- 0
performa/tests/atop_sample.txt View File

@@ -0,0 +1,50 @@
1
+RESET
2
+CPU host 1456480863 2016/02/26 10:01:03 2503659 100 4 5355049 7555867 5 984566626 2589741 63689 0 0 0 12768 100
3
+cpu host 1456480863 2016/02/26 10:01:03 2503659 100 0 1705191 2253800 0 243472217 2465631 12723 0 0 0 3192 100
4
+SEP
5
+CPU host 1456480864 2016/02/26 10:01:04 1 100 4 4 4 0 392 0 0 0 0 0 12768 100
6
+cpu host 1456480864 2016/02/26 10:01:04 1 100 0 0 0 0 99 0 0 0 0 0 3192 100
7
+cpu host 1456480864 2016/02/26 10:01:04 1 100 1 1 0 0 99 0 0 0 0 0 3192 100
8
+cpu host 1456480864 2016/02/26 10:01:04 1 100 2 2 3 0 95 0 0 0 0 0 3192 100
9
+cpu host 1456480864 2016/02/26 10:01:04 1 100 3 0 0 0 100 0 0 0 0 0 3192 100
10
+MEM host 1456480864 2016/02/26 10:01:04 1 4096 2044208 893540 809906 85798 72782 0
11
+NET host 1456480864 2016/02/26 10:01:04 1 upper 0 0 0 0 0 0 0 0
12
+NET host 1456480864 2016/02/26 10:01:04 1 ovs-system 0 0 0 0 0 0
13
+NET host 1456480864 2016/02/26 10:01:04 1 br-int 0 0 0 0 0 0
14
+NET host 1456480864 2016/02/26 10:01:04 1 eth0 0 0 0 0 100 1
15
+NET host 1456480864 2016/02/26 10:01:04 1 lo 0 0 0 0 0 0
16
+NET host 1456480864 2016/02/26 10:01:04 1 br-ex 0 0 0 0 0 0
17
+NET host 1456480864 2016/02/26 10:01:04 1 virbr0 0 0 0 0 0 0
18
+PRC host 1456480864 2016/02/26 10:01:04 1 8595 (epmd) S 100 0 1 0 120 0 0 0 0
19
+PRC host 1456480864 2016/02/26 10:01:04 1 8634 (beam.smp) S 100 1 1 0 120 0 0 1 0
20
+PRC host 1456480864 2016/02/26 10:01:04 1 11014 (dstat) S 100 1 2 0 120 0 0 2 0
21
+PRC host 1456480864 2016/02/26 10:01:04 1 14134 (glance-api) S 100 1 0 0 120 0 0 0 0
22
+PRC host 1456480864 2016/02/26 10:01:04 1 19929 (atop) R 100 1 2 0 120 0 0 2 0
23
+PRM host 1456480864 2016/02/26 10:01:04 1 8595 (epmd) S 4096 7492 316 0 0 0 0 0
24
+PRM host 1456480864 2016/02/26 10:01:04 1 8634 (beam.smp) S 4096 2168928 51548 0 0 0 0 0
25
+PRM host 1456480864 2016/02/26 10:01:04 1 11014 (dstat) S 4096 34044 7472 2796 0 0 0 0
26
+PRM host 1456480864 2016/02/26 10:01:04 1 14134 (glance-api) S 4096 219848 87044 2796 0 0 0 0
27
+PRM host 1456480864 2016/02/26 10:01:04 1 19929 (atop) R 4096 17004 1972 148 244 364 874 0
28
+SEP
29
+CPU host 1456480865 2016/02/26 10:01:05 1 100 4 4 4 0 392 0 0 0 0 0 12768 100
30
+cpu host 1456480865 2016/02/26 10:01:05 1 100 0 2 1 0 97 0 0 0 0 0 3192 100
31
+cpu host 1456480865 2016/02/26 10:01:05 1 100 1 0 0 0 100 0 0 0 0 0 3192 100
32
+cpu host 1456480865 2016/02/26 10:01:05 1 100 2 3 3 0 94 0 0 0 0 0 3192 100
33
+cpu host 1456480865 2016/02/26 10:01:05 1 100 3 0 0 0 100 0 0 0 0 0 3192 100
34
+MEM host 1456480865 2016/02/26 10:01:05 1 4096 2044208 893540 809909 85798 72782 0
35
+NET host 1456480865 2016/02/26 10:01:05 1 upper 0 0 0 0 0 0 0 0
36
+NET host 1456480865 2016/02/26 10:01:05 1 ovs-system 0 0 0 0 0 0
37
+NET host 1456480865 2016/02/26 10:01:05 1 br-int 0 0 0 0 0 0
38
+NET host 1456480865 2016/02/26 10:01:05 1 eth0 0 0 0 0 100 1
39
+NET host 1456480865 2016/02/26 10:01:05 1 lo 0 0 0 0 0 0
40
+NET host 1456480865 2016/02/26 10:01:05 1 br-ex 0 0 0 0 0 0
41
+NET host 1456480865 2016/02/26 10:01:05 1 virbr0 0 0 0 0 0 0
42
+PRC host 1456480865 2016/02/26 10:01:05 1 7315 (ovs-vswitchd) S 100 0 1 -10 110 0 0 0 0
43
+PRC host 1456480865 2016/02/26 10:01:05 1 11014 (dstat) S 100 2 0 0 120 0 0 2 0
44
+PRC host 1456480865 2016/02/26 10:01:05 1 14134 (glance-api) S 100 1 0 0 120 0 0 0 0
45
+PRC host 1456480865 2016/02/26 10:01:05 1 19929 (atop) R 100 1 2 0 120 0 0 2 0
46
+PRM host 1456480865 2016/02/26 10:01:05 1 7315 (ovs-vswitchd) S 4096 243088 32016 0 0 0 0 0
47
+PRM host 1456480865 2016/02/26 10:01:05 1 11014 (dstat) S 4096 34044 7472 2796 0 0 0 0
48
+PRM host 1456480865 2016/02/26 10:01:05 1 14134 (glance-api) S 4096 219848 87044 2796 0 0 0 0
49
+PRM host 1456480865 2016/02/26 10:01:05 1 19929 (atop) R 4096 17004 1972 148 0 0 751 0
50
+SEP

+ 90
- 0
performa/tests/test_atop.py View File

@@ -0,0 +1,90 @@
1
+# Copyright (c) 2016 OpenStack Foundation
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License");
4
+# you may not use this file except in compliance with the License.
5
+# You may obtain a copy of the License at
6
+#
7
+#   http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS,
11
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12
+# implied.
13
+# See the License for the specific language governing permissions and
14
+# limitations under the License.
15
+
16
+import testtools
17
+
18
+from performa.modules import atop
19
+
20
+
21
+def _read_sample():
22
+    with open('performa/tests/atop_sample.txt') as f:
23
+        return f.read()
24
+
25
+
26
+class TestAtop(testtools.TestCase):
27
+    def test_parse_cpu_total(self):
28
+        expected = [{'cpu_count': 4, 'date': '2016/02/26', 'guest': 0.0,
29
+                     'host': 'host', 'idle': 3.92, 'interval': 1, 'irq': 0.0,
30
+                     'label': 'CPU', 'nice': 0.0, 'softirq': 0.0, 'steal': 0.0,
31
+                     'sys': 0.04, 'ticks_per_second': 100, 'time': '10:01:04',
32
+                     'timestamp': 1456480864, 'user': 0.04, 'wait': 0.0},
33
+                    {'cpu_count': 4, 'date': '2016/02/26', 'guest': 0.0,
34
+                     'host': 'host', 'idle': 3.92, 'interval': 1, 'irq': 0.0,
35
+                     'label': 'CPU', 'nice': 0.0, 'softirq': 0.0, 'steal': 0.0,
36
+                     'sys': 0.04, 'ticks_per_second': 100, 'time': '10:01:05',
37
+                     'timestamp': 1456480865, 'user': 0.04, 'wait': 0.0}]
38
+
39
+        self.assertEqual(expected, atop.parse_output(_read_sample(), ['CPU']))
40
+
41
+    def test_parse_cpu(self):
42
+        needle = {'cpu_id': 2, 'date': '2016/02/26', 'guest': 0.0,
43
+                  'host': 'host', 'idle': 0.94, 'interval': 1, 'irq': 0.0,
44
+                  'label': 'cpu', 'nice': 0.0, 'softirq': 0.0, 'steal': 0.0,
45
+                  'sys': 0.03, 'ticks_per_second': 100, 'time': '10:01:05',
46
+                  'timestamp': 1456480865, 'user': 0.03, 'wait': 0.0}
47
+
48
+        self.assertIn(needle, atop.parse_output(_read_sample(), ['cpu']))
49
+
50
+    def test_parse_mem(self):
51
+        expected = [
52
+            {'buffer': 351428608, 'cache': 3317374976, 'date': '2016/02/26',
53
+             'dirty': 0, 'free': 3659939840, 'host': 'host', 'interval': 1,
54
+             'label': 'MEM', 'page_size': 4096, 'phys': 8373075968,
55
+             'slab': 298115072, 'time': '10:01:04', 'timestamp': 1456480864},
56
+            {'buffer': 351428608, 'cache': 3317387264, 'date': '2016/02/26',
57
+             'dirty': 0, 'free': 3659939840, 'host': 'host', 'interval': 1,
58
+             'label': 'MEM', 'page_size': 4096, 'phys': 8373075968,
59
+             'slab': 298115072, 'time': '10:01:05', 'timestamp': 1456480865}]
60
+
61
+        self.assertEqual(expected, atop.parse_output(_read_sample(), ['MEM']))
62
+
63
+    def test_parse_net(self):
64
+        needle = {'date': '2016/02/26', 'host': 'host', 'interval': 1,
65
+                  'ip_dx': 0, 'ip_fx': 0, 'ip_rx': 0, 'ip_tx': 0,
66
+                  'label': 'NET', 'tcp_rx': 0, 'tcp_tx': 0, 'time': '10:01:04',
67
+                  'timestamp': 1456480864, 'udp_rx': 0, 'udp_tx': 0}
68
+
69
+        self.assertIn(needle, atop.parse_output(_read_sample(), ['NET']))
70
+
71
+    def test_parse_prc(self):
72
+        needle = {'current_cpu': 2, 'date': '2016/02/26', 'host': 'host',
73
+                  'interval': 1, 'label': 'PRC', 'name': 'dstat', 'nice': 0,
74
+                  'pid': 11014, 'priority': 120, 'realtime_priority': 0,
75
+                  'scheduling_policy': 0, 'sleep_avg': 0, 'state': 'S',
76
+                  'sys': 0.02, 'ticks_per_second': 100, 'time': '10:01:04',
77
+                  'timestamp': 1456480864, 'user': 0.01}
78
+
79
+        self.assertIn(needle, atop.parse_output(_read_sample(), ['PRC']))
80
+
81
+    def test_parse_prm(self):
82
+        needle = {'date': '2016/02/26', 'host': 'host', 'interval': 1,
83
+                  'label': 'PRM', 'major_page_faults': 0,
84
+                  'minor_page_faults': 751, 'name': 'atop', 'page_size': 4096,
85
+                  'pid': 19929, 'resident': 2019328, 'resident_growth': 0,
86
+                  'shared': 151552, 'state': 'R', 'time': '10:01:05',
87
+                  'timestamp': 1456480865, 'virtual': 17412096,
88
+                  'virtual_growth': 0}
89
+
90
+        self.assertIn(needle, atop.parse_output(_read_sample(), ['PRM']))

Loading…
Cancel
Save