Browse Source

Merge "Implement format auto-detection"

Zuul 1 year ago
parent
commit
18985c7485
1 changed files with 65 additions and 12 deletions
  1. 65
    12
      oslogmerger/oslogmerger.py

+ 65
- 12
oslogmerger/oslogmerger.py View File

@@ -152,6 +152,8 @@ class TSLogParser(LogParser):
152 152
         stat = os.stat(filename)
153 153
         mtime = datetime.fromtimestamp(stat.st_mtime)
154 154
         timestamp = self._get_last_timestamp(filename)
155
+        if timestamp is None:
156
+            raise ValueError("Didn't find timestamp")
155 157
         self.start_date = mtime - timedelta(seconds=timestamp)
156 158
 
157 159
     @classmethod
@@ -187,10 +189,45 @@ class TSLogParser(LogParser):
187 189
 
188 190
 
189 191
 class LogFile(object):
190
-    def __init__(self, filename, alias, parser_cls):
192
+    def _detect_format(self, filename):
191 193
         self.open(filename)
194
+
195
+        parsers = []
196
+        for cls in LOG_TYPES.values():
197
+            if cls is None:
198
+                continue
199
+
200
+            try:
201
+                parsers.append(cls(filename))
202
+            except ValueError:
203
+                # Don't consider the parser if we can't instantiate it for this
204
+                # file
205
+                pass
206
+
207
+        # Try to parse the first few lines with each parser in turn, returning
208
+        # the first to successfully parse a line
209
+        for i in range(0, 5):
210
+            line = self._readline()
211
+            for parser in parsers:
212
+                try:
213
+                    parser.parse_line(line)
214
+
215
+                    # It worked!
216
+                    return parser
217
+                except ValueError:
218
+                    pass
219
+
220
+        raise ValueError("Failed to detect format of %s" % self.alias)
221
+
222
+    def __init__(self, filename, alias, parser_cls=None):
192 223
         self.alias = alias
193
-        self.parser = parser_cls(filename)
224
+
225
+        if parser_cls is None:
226
+            self.parser = self._detect_format(filename)
227
+        else:
228
+            self.parser = parser_cls(filename)
229
+
230
+        self.open(filename)
194 231
 
195 232
     def open(self, filename):
196 233
         self._filename = filename
@@ -234,12 +271,18 @@ class LogFile(object):
234 271
         self.next_entry = None
235 272
         return self
236 273
 
237
-    def _readline(self, entry):
274
+    def _readline(self):
275
+        line = self._file.readline()
276
+        if line == "":
277
+            return None
278
+        line.replace('\0', ' ')
279
+        return line
280
+
281
+    def _next_entry(self, entry):
238 282
         while True:
239
-            line = self._file.readline()
240
-            if line == "":
283
+            line = self._readline()
284
+            if line is None:
241 285
                 return entry, None
242
-            line.replace('\0', ' ')
243 286
 
244 287
             try:
245 288
                 dt, dt_str, data = self.parser.parse_line(line)
@@ -258,7 +301,7 @@ class LogFile(object):
258 301
         return self.next()
259 302
 
260 303
     def next(self):
261
-        self.entry, self.next_entry = self._readline(self.next_entry)
304
+        self.entry, self.next_entry = self._next_entry(self.next_entry)
262 305
         if self.entry is None:
263 306
             raise StopIteration()
264 307
         return self.entry
@@ -278,7 +321,8 @@ class LogFile(object):
278 321
 
279 322
 
280 323
 LOG_TYPES = {
281
-    'logfiles': OSLogParser,
324
+    'logfiles_detect': None,
325
+    'logfiles_o': OSLogParser,
282 326
     'logfiles_m': MsgLogParser,
283 327
     'logfiles_t': TSLogParser,
284 328
 }
@@ -302,8 +346,13 @@ def process_logs(cfg):
302 346
     # now, though.
303 347
     aliases = generate_aliases(paths_aliases, cfg)
304 348
 
305
-    logs = [LogFile(path, aliases[path], parser_cls)
306
-            for path, parser_cls in paths_parsers.items()]
349
+    logs = []
350
+    for path, parser_cls in paths_parsers.items():
351
+        try:
352
+            logs.append(LogFile(path, aliases[path], parser_cls))
353
+        except ValueError:
354
+            print('WARNING: %s unable to determine format, ignoring' % path,
355
+                  file=sys.stderr)
307 356
 
308 357
     entry_iters = [iter(log) for log in logs]
309 358
     for entry in heapq.merge(*entry_iters):
@@ -551,14 +600,18 @@ one has not been provided:'
551 600
                         help='Base path for all the log files')
552 601
     parser.add_argument('--log-postfix ', '-p', dest='log_postfix',
553 602
                         help='Append to all the log files path')
554
-    parser.add_argument('logfiles', nargs='+', metavar='log_file[:ALIAS]',
555
-                        help='OpenStack log file.')
603
+    parser.add_argument('logfiles_detect', nargs='+',
604
+                        metavar='log_file[:ALIAS]',
605
+                        help='Log file (auto-detect format)')
556 606
     parser.add_argument('--alias-level', '-a', type=int, default=0,
557 607
                         dest='alias_level',
558 608
                         help='Level of smart alias naming (0-3)')
559 609
     parser.add_argument('--min-memory', '-m', default=False,
560 610
                         action='store_true', dest='limit_memory',
561 611
                         help='This option is deprecated and has no effect')
612
+    parser.add_argument('--os-logs', '-ol', default=[], nargs='+',
613
+                        dest='logfiles_o', metavar='file[:ALIAS]',
614
+                        help='Openstack log files')
562 615
     parser.add_argument('--msg-logs', '-ml', default=[], nargs='+',
563 616
                         dest='logfiles_m', metavar='file[:ALIAS]',
564 617
                         help='Message log files with format: Oct 15 14:11:19')

Loading…
Cancel
Save