Browse Source

Added a new probe: netprobe

This is an standalone .py file that can be uploaded to remote
servers and used to log network packets over network interfaces.

It will discover new interfaces automatically, in the root or
inside network namespaces.

Interfaces and namespaces are filtered via regular expressions.

usage: netprobe [-h] [-v] [--netns-re NETNS_REGEX]
                [--netdev-re NETDEV_REGEX]
                [--tcpdump-filter TCPDUMP_FILTER]
                [--check-interval CHECK_INTERVAL]

This tool will track system network devices as they appear in a host,
and start tcpdump processes for each of them, while the output of all
the tcpdumps goes in a single openstack-like log.

optional arguments:
  -h, --help            show this help message and exit
  -v, --version         show program's version number and exit
  --netns-re NETNS_REGEX, -n NETNS_REGEX
  --netdev-re NETDEV_REGEX, -d NETDEV_REGEX
  --tcpdump-filter TCPDUMP_FILTER, -t TCPDUMP_FILTER
  --check-interval CHECK_INTERVAL, -i CHECK_INTERVAL
                        The interval between interface checks
Miguel Angel Ajo 3 years ago
parent
commit
7eb4f383db
5 changed files with 197 additions and 0 deletions
  1. 46
    0
      README.rst
  2. 1
    0
      contrib/probes/netprobe.py
  3. 0
    0
      oslogmerger/probes/__init__.py
  4. 149
    0
      oslogmerger/probes/netprobe.py
  5. 1
    0
      setup.py

+ 46
- 0
README.rst View File

@@ -216,3 +216,49 @@ Would result in::
216 216
     2016-02-01 10:25:34.700 [1/C-VOL] ...
217 217
     2016-02-01 10:26:34.710 [1/N-API] ...
218 218
     2016-02-01 10:27:34.680 [2/N-CPU] ...
219
+
220
+System probes
221
+=============
222
+A set of system probes are provided as companion tools to help debugging
223
+common issues.
224
+
225
+netprobe
226
+~~~~~~~~
227
+Under the probes directory netprobe.py can be found, this tool depends on
228
+tcpdump being available on the host.
229
+
230
+This probe will inspect the system for new UP network devices periodically,
231
+and when found, a tcpdump will be started with the configured filter,
232
+logging everything on the openstack log format.
233
+
234
+The net namespaces are not filtered by default.
235
+
236
+The network interfaces default filter is::
237
+
238
+    tap.*|qbr.*|qg-\.*|qr-\.*
239
+
240
+
241
+The default packet filter is::
242
+
243
+    (arp or rarp) or (udp and (port 67 or port 68)) or icmp or icmp6
244
+
245
+
246
+Usage details::
247
+
248
+    usage: netprobe [-h] [-v] [--netns-re NETNS_REGEX] [--netdev-re NETDEV_REGEX]
249
+                    [--tcpdump-filter TCPDUMP_FILTER]
250
+                    [--check-interval CHECK_INTERVAL]
251
+
252
+    This tool will track system network devices as they appear in a host,
253
+    and start tcpdump processes for each of them, while the output of all
254
+    the tcpdumps goes in a single openstack-like log.
255
+
256
+    optional arguments:
257
+      -h, --help            show this help message and exit
258
+      -v, --version         show program's version number and exit
259
+      --netns-re NETNS_REGEX, -n NETNS_REGEX
260
+      --netdev-re NETDEV_REGEX, -d NETDEV_REGEX
261
+      --tcpdump-filter TCPDUMP_FILTER, -t TCPDUMP_FILTER
262
+      --check-interval CHECK_INTERVAL, -i CHECK_INTERVAL
263
+                            The interval between interface checks
264
+

+ 1
- 0
contrib/probes/netprobe.py View File

@@ -0,0 +1 @@
1
+../../oslogmerger/probes/netprobe.py

+ 0
- 0
oslogmerger/probes/__init__.py View File


+ 149
- 0
oslogmerger/probes/netprobe.py View File

@@ -0,0 +1,149 @@
1
+#!/usr/bin/env python
2
+from __future__ import print_function
3
+
4
+import argparse
5
+import datetime
6
+import re
7
+import sys
8
+import subprocess
9
+import time
10
+import threading
11
+
12
+__version__ = '0.0.1'
13
+
14
+INTERFACE_RE = re.compile('\d+: (.+):')
15
+DEFAULT_CHECK_INTERVAL = 5
16
+DEFAULT_INTERFACE_RE = "tap.*|qbr.*|qg-\.*|qr-\.*"
17
+DEFAULT_TCPDUMP_FILTER = '(arp or rarp) or (udp and (port 67 or port 68))' + \
18
+                         ' or icmp or icmp6'
19
+
20
+
21
+def netns():
22
+    return filter(lambda line: len(line) > 0,
23
+                  subprocess.check_output(['ip', 'netns']).split('\n'))
24
+
25
+
26
+def _netns_cmd(netns=None):
27
+    if netns:
28
+        return ['ip', 'netns', 'exec', netns]
29
+    else:
30
+        return []
31
+
32
+
33
+def interfaces(netns=None):
34
+    cmd = _netns_cmd(netns) + ['ip', 'link']
35
+    out = subprocess.check_output(cmd).split('\n')
36
+    not_down = "\n".join(filter(lambda line: line.find(' DOWN ') == -1, out))
37
+    return INTERFACE_RE.findall(not_down)
38
+
39
+
40
+def _time_now():
41
+    return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
42
+
43
+
44
+def _date_now():
45
+    return datetime.datetime.now().strftime('%Y-%m-%d')
46
+
47
+
48
+def spawn_tcpdump(interface, netns=None,
49
+                  filters=DEFAULT_TCPDUMP_FILTER):
50
+    cmd = _netns_cmd(netns) + ['tcpdump', '-i', interface, '-n', '-e', '-l']
51
+    cmd += [filters]
52
+    tcpdump = subprocess.Popen(cmd, bufsize=0,
53
+                               stdout=subprocess.PIPE,
54
+                               stderr=subprocess.PIPE)
55
+    print(_time_now(), interface,
56
+          "started dumping with filter {}".format(filters))
57
+    while True:
58
+        out = tcpdump.stdout.readline()
59
+        if out == '':
60
+            break
61
+        line = out.rstrip()
62
+        chunks = line.split(' ')
63
+        timestamp = chunks[0]
64
+        tcpdump_trace = ' '.join(chunks[1:])
65
+        # FIXME: _date_now + tcpdump_timestamp can be raceful at the
66
+        #        end of the day
67
+        print(_date_now(), timestamp, interface, tcpdump_trace)
68
+
69
+
70
+def parse_args():
71
+    class MyParser(argparse.ArgumentParser):
72
+        """Class to print verbose help on error."""
73
+        def error(self, message):
74
+            self.print_help()
75
+            sys.stderr.write('\nerror: %s\n' % message)
76
+            sys.exit(2)
77
+
78
+    general_description = """
79
+This tool will track system network devices as they appear in a host,
80
+and start tcpdump processes for each of them, while the output of all
81
+the tcpdumps goes in a single openstack-like log.
82
+"""
83
+
84
+    general_epilog = ""
85
+
86
+    parser = MyParser(description=general_description, version=__version__,
87
+                      epilog=general_epilog, argument_default='',
88
+                      formatter_class=argparse.RawTextHelpFormatter)
89
+    parser.add_argument('--netns-re', '-n', dest='netns_regex',
90
+                        help='')
91
+    parser.add_argument('--netdev-re', '-d', dest='netdev_regex',
92
+                        help='', default=DEFAULT_INTERFACE_RE)
93
+    parser.add_argument('--tcpdump-filter', '-t', dest='tcpdump_filter',
94
+                        help='',
95
+                        default=DEFAULT_TCPDUMP_FILTER)
96
+    parser.add_argument('--check-interval', '-i', type=int,
97
+                        default=DEFAULT_CHECK_INTERVAL,
98
+                        dest='check_interval',
99
+                        help='The interval between interface checks')
100
+
101
+    return parser.parse_args()
102
+
103
+
104
+def create_tcpdump_thread(interface, namespace, tcpdump_filter, thread_name):
105
+    thread = threading.Thread(target=spawn_tcpdump,
106
+                              name=thread_name,
107
+                              kwargs={'interface': interface,
108
+                                      'netns': namespace,
109
+                                      'filters': tcpdump_filter})
110
+    thread.start()
111
+    return thread
112
+
113
+
114
+def scan_loop(args):
115
+    tracked_ifs = {}
116
+    netns_re = re.compile(args.netns_regex)
117
+    netdev_re = re.compile(args.netdev_regex)
118
+    tcpdump_filter = args.tcpdump_filter
119
+    while True:
120
+        print(_time_now(), "checking interfaces", file=sys.stderr)
121
+        namespaces = filter(netns_re.match, netns()) + [None]
122
+        for namespace in namespaces:
123
+            ifs = filter(netdev_re.match, interfaces(namespace))
124
+            for interface in ifs:
125
+                name = "{} (@ns {})".format(interface, namespace)
126
+                if name not in tracked_ifs:
127
+                    print(_time_now(),
128
+                          "Watching interface {}".format(name),
129
+                          file=sys.stderr)
130
+                    tracked_ifs[name] = create_tcpdump_thread(interface,
131
+                                                              namespace,
132
+                                                              tcpdump_filter,
133
+                                                              name)
134
+        # tcpdump thread go away when interface is removed
135
+        for thread_name, thread in tracked_ifs.items():
136
+            if not thread.is_alive():
137
+                print(_time_now(),
138
+                      "Interface {} went away".format(thread_name))
139
+                tracked_ifs.pop(thread_name).join()
140
+
141
+        time.sleep(args.check_interval)
142
+
143
+
144
+def main():
145
+    args = parse_args()
146
+    scan_loop(args)
147
+
148
+if __name__ == '__main__':
149
+    main()

+ 1
- 0
setup.py View File

@@ -66,6 +66,7 @@ setup(
66 66
         'console_scripts': [
67 67
             'os-log-merger=oslogmerger.oslogmerger:main',
68 68
             'oslogmerger=oslogmerger.oslogmerger:main',
69
+	    'netprobe=oslogmerger.probes.netprobe:main',
69 70
         ],
70 71
     },
71 72
 )

Loading…
Cancel
Save