Implement format auto-detection
Add a new option --os-logs/-ol to explicitly specify OpenStack format logs. Logs whose format is not given explicitly are auto-detected. Change-Id: Ie97127f24f34ab386138db28b626abf4c259ac24
This commit is contained in:
parent
27599915cb
commit
06de0ba054
|
@ -152,6 +152,8 @@ class TSLogParser(LogParser):
|
||||||
stat = os.stat(filename)
|
stat = os.stat(filename)
|
||||||
mtime = datetime.fromtimestamp(stat.st_mtime)
|
mtime = datetime.fromtimestamp(stat.st_mtime)
|
||||||
timestamp = self._get_last_timestamp(filename)
|
timestamp = self._get_last_timestamp(filename)
|
||||||
|
if timestamp is None:
|
||||||
|
raise ValueError("Didn't find timestamp")
|
||||||
self.start_date = mtime - timedelta(seconds=timestamp)
|
self.start_date = mtime - timedelta(seconds=timestamp)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
@ -187,10 +189,45 @@ class TSLogParser(LogParser):
|
||||||
|
|
||||||
|
|
||||||
class LogFile(object):
|
class LogFile(object):
|
||||||
def __init__(self, filename, alias, parser_cls):
|
def _detect_format(self, filename):
|
||||||
self.open(filename)
|
self.open(filename)
|
||||||
|
|
||||||
|
parsers = []
|
||||||
|
for cls in LOG_TYPES.values():
|
||||||
|
if cls is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
parsers.append(cls(filename))
|
||||||
|
except ValueError:
|
||||||
|
# Don't consider the parser if we can't instantiate it for this
|
||||||
|
# file
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Try to parse the first few lines with each parser in turn, returning
|
||||||
|
# the first to successfully parse a line
|
||||||
|
for i in range(0, 5):
|
||||||
|
line = self._readline()
|
||||||
|
for parser in parsers:
|
||||||
|
try:
|
||||||
|
parser.parse_line(line)
|
||||||
|
|
||||||
|
# It worked!
|
||||||
|
return parser
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
raise ValueError("Failed to detect format of %s" % self.alias)
|
||||||
|
|
||||||
|
def __init__(self, filename, alias, parser_cls=None):
|
||||||
self.alias = alias
|
self.alias = alias
|
||||||
self.parser = parser_cls(filename)
|
|
||||||
|
if parser_cls is None:
|
||||||
|
self.parser = self._detect_format(filename)
|
||||||
|
else:
|
||||||
|
self.parser = parser_cls(filename)
|
||||||
|
|
||||||
|
self.open(filename)
|
||||||
|
|
||||||
def open(self, filename):
|
def open(self, filename):
|
||||||
self._filename = filename
|
self._filename = filename
|
||||||
|
@ -234,12 +271,18 @@ class LogFile(object):
|
||||||
self.next_entry = None
|
self.next_entry = None
|
||||||
return self
|
return self
|
||||||
|
|
||||||
def _readline(self, entry):
|
def _readline(self):
|
||||||
|
line = self._file.readline()
|
||||||
|
if line == "":
|
||||||
|
return None
|
||||||
|
line.replace('\0', ' ')
|
||||||
|
return line
|
||||||
|
|
||||||
|
def _next_entry(self, entry):
|
||||||
while True:
|
while True:
|
||||||
line = self._file.readline()
|
line = self._readline()
|
||||||
if line == "":
|
if line is None:
|
||||||
return entry, None
|
return entry, None
|
||||||
line.replace('\0', ' ')
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
dt, dt_str, data = self.parser.parse_line(line)
|
dt, dt_str, data = self.parser.parse_line(line)
|
||||||
|
@ -258,7 +301,7 @@ class LogFile(object):
|
||||||
return self.next()
|
return self.next()
|
||||||
|
|
||||||
def next(self):
|
def next(self):
|
||||||
self.entry, self.next_entry = self._readline(self.next_entry)
|
self.entry, self.next_entry = self._next_entry(self.next_entry)
|
||||||
if self.entry is None:
|
if self.entry is None:
|
||||||
raise StopIteration()
|
raise StopIteration()
|
||||||
return self.entry
|
return self.entry
|
||||||
|
@ -278,7 +321,8 @@ class LogFile(object):
|
||||||
|
|
||||||
|
|
||||||
LOG_TYPES = {
|
LOG_TYPES = {
|
||||||
'logfiles': OSLogParser,
|
'logfiles_detect': None,
|
||||||
|
'logfiles_o': OSLogParser,
|
||||||
'logfiles_m': MsgLogParser,
|
'logfiles_m': MsgLogParser,
|
||||||
'logfiles_t': TSLogParser,
|
'logfiles_t': TSLogParser,
|
||||||
}
|
}
|
||||||
|
@ -302,8 +346,13 @@ def process_logs(cfg):
|
||||||
# now, though.
|
# now, though.
|
||||||
aliases = generate_aliases(paths_aliases, cfg)
|
aliases = generate_aliases(paths_aliases, cfg)
|
||||||
|
|
||||||
logs = [LogFile(path, aliases[path], parser_cls)
|
logs = []
|
||||||
for path, parser_cls in paths_parsers.items()]
|
for path, parser_cls in paths_parsers.items():
|
||||||
|
try:
|
||||||
|
logs.append(LogFile(path, aliases[path], parser_cls))
|
||||||
|
except ValueError:
|
||||||
|
print('WARNING: %s unable to determine format, ignoring' % path,
|
||||||
|
file=sys.stderr)
|
||||||
|
|
||||||
entry_iters = [iter(log) for log in logs]
|
entry_iters = [iter(log) for log in logs]
|
||||||
for entry in heapq.merge(*entry_iters):
|
for entry in heapq.merge(*entry_iters):
|
||||||
|
@ -551,14 +600,18 @@ one has not been provided:'
|
||||||
help='Base path for all the log files')
|
help='Base path for all the log files')
|
||||||
parser.add_argument('--log-postfix ', '-p', dest='log_postfix',
|
parser.add_argument('--log-postfix ', '-p', dest='log_postfix',
|
||||||
help='Append to all the log files path')
|
help='Append to all the log files path')
|
||||||
parser.add_argument('logfiles', nargs='+', metavar='log_file[:ALIAS]',
|
parser.add_argument('logfiles_detect', nargs='+',
|
||||||
help='OpenStack log file.')
|
metavar='log_file[:ALIAS]',
|
||||||
|
help='Log file (auto-detect format)')
|
||||||
parser.add_argument('--alias-level', '-a', type=int, default=0,
|
parser.add_argument('--alias-level', '-a', type=int, default=0,
|
||||||
dest='alias_level',
|
dest='alias_level',
|
||||||
help='Level of smart alias naming (0-3)')
|
help='Level of smart alias naming (0-3)')
|
||||||
parser.add_argument('--min-memory', '-m', default=False,
|
parser.add_argument('--min-memory', '-m', default=False,
|
||||||
action='store_true', dest='limit_memory',
|
action='store_true', dest='limit_memory',
|
||||||
help='This option is deprecated and has no effect')
|
help='This option is deprecated and has no effect')
|
||||||
|
parser.add_argument('--os-logs', '-ol', default=[], nargs='+',
|
||||||
|
dest='logfiles_o', metavar='file[:ALIAS]',
|
||||||
|
help='Openstack log files')
|
||||||
parser.add_argument('--msg-logs', '-ml', default=[], nargs='+',
|
parser.add_argument('--msg-logs', '-ml', default=[], nargs='+',
|
||||||
dest='logfiles_m', metavar='file[:ALIAS]',
|
dest='logfiles_m', metavar='file[:ALIAS]',
|
||||||
help='Message log files with format: Oct 15 14:11:19')
|
help='Message log files with format: Oct 15 14:11:19')
|
||||||
|
|
Loading…
Reference in New Issue