From bbef755ed80d9e7a5812b01f5e4fd6b70bcb0605 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Tue, 11 Apr 2017 17:01:45 -0400 Subject: [PATCH] Fix systemd line parsing There was a bug in systemd parsing with a missing space so that it was never matching. That's fixed. In addition, the systemd and syslog line parsing are close enough to each other, that in our previous patterns they would match each other's lines. The important distinction is that systemd includes [pid] after the host which syslog doesn't. We also have to realize that status might not be in a systemd line, and that's ok, but still match it, given that otherwise we run the risk of falling into the syslog match. Change-Id: Id91c167118894c86b9b0d875a1bb36bdbe239ebd --- os_loganalyze/filter.py | 10 +++++---- .../samples/devstack@c-api.service.log.txt | 10 +++++++++ os_loganalyze/tests/test_filters.py | 22 +++++++++++++++++++ 3 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 os_loganalyze/tests/samples/devstack@c-api.service.log.txt diff --git a/os_loganalyze/filter.py b/os_loganalyze/filter.py index 467657f..568076b 100644 --- a/os_loganalyze/filter.py +++ b/os_loganalyze/filter.py @@ -37,10 +37,11 @@ STATUSFMT = '(DEBUG|INFO|WARNING|ERROR|TRACE|AUDIT)' OSLO_LOGMATCH = '^(?P%s)(?P(?P \d+)? (?P%s).*)' % \ (DATEFMT, STATUSFMT) SYSLOG_MATCH = ('^(?P%s)(?P (?P[\w\-]+) ' - '(?P\S+):.*)' % + '(?P[^\[\s]+):.*)' % (SYSLOGDATE)) -SYSTEMD_MATCH = '^(?P%s)(?P\S+ \S+\: (?P%s).*)' % \ - (SYSLOGDATE, STATUSFMT) +SYSTEMD_MATCH = ( + '^(?P%s) (?P(?P\S+) \S+\[\d+\]\: (?P%s)?.*)' % + (SYSLOGDATE, STATUSFMT)) CONSOLE_MATCH = '^(?P%s)(?P.*)' % DATEFMT OSLORE = re.compile(OSLO_LOGMATCH) @@ -88,9 +89,10 @@ class LogLine(object): return m = SYSTEMDRE.match(line) if m: - self.status = m.group('status') + self.status = m.group('status') or "NONE" self.line = m.group('line') self.date = m.group('date') + self.host = m.group('host') return m = CONSOLERE.match(line) if m: diff --git a/os_loganalyze/tests/samples/devstack@c-api.service.log.txt b/os_loganalyze/tests/samples/devstack@c-api.service.log.txt new file mode 100644 index 0000000..ef0db52 --- /dev/null +++ b/os_loganalyze/tests/samples/devstack@c-api.service.log.txt @@ -0,0 +1,10 @@ +-- Logs begin at Tue 2017-03-28 12:02:58 UTC, end at Tue 2017-03-28 13:16:45 UTC. -- +Mar 28 12:20:42.377230 ubuntu-xenial-osic-cloud1-s3500-8127579 systemd[1]: Started Devstack devstack@c-api.service. +Mar 28 12:20:43.570064 ubuntu-xenial-osic-cloud1-s3500-8127579 cinder-api[8526]: WARNING oslo_reports.guru_meditation_report [-] Guru meditation now registers SIGUSR1 and SIGUSR2 by default for backward compatibility. SIGUSR1 will no longer be registered in a future release, so please use SIGUSR2 to generate reports. +Mar 28 12:20:44.172534 ubuntu-xenial-osic-cloud1-s3500-8127579 cinder-api[8526]: DEBUG oslo_concurrency.lockutils [-] Acquired semaphore "singleton_lock" from (pid=8526) lock /usr/local/lib/python2.7/dist-packages/oslo_concurrency/lockutils.py:212 +Mar 28 12:20:44.173011 ubuntu-xenial-osic-cloud1-s3500-8127579 cinder-api[8526]: DEBUG oslo_concurrency.lockutils [-] Releasing semaphore "singleton_lock" from (pid=8526) lock /usr/local/lib/python2.7/dist-packages/oslo_concurrency/lockutils.py:225 +Mar 28 12:20:44.173575 ubuntu-xenial-osic-cloud1-s3500-8127579 cinder-api[8526]: DEBUG oslo.service.wsgi [-] Loading app osapi_volume from /etc/cinder/api-paste.ini from (pid=8526) load_app /usr/local/lib/python2.7/dist-packages/oslo_service/wsgi.py:352 +Mar 28 12:20:44.278200 ubuntu-xenial-osic-cloud1-s3500-8127579 cinder-api[8526]: INFO root [-] Generating grammar tables from /usr/lib/python2.7/lib2to3/Grammar.txt +Mar 28 12:20:44.301267 ubuntu-xenial-osic-cloud1-s3500-8127579 cinder-api[8526]: INFO root [-] Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt +Mar 28 12:20:44.425018 ubuntu-xenial-osic-cloud1-s3500-8127579 cinder-api[8526]: INFO cinder.api.extensions [-] Initializing extension manager. +Mar 28 12:20:44.426439 ubuntu-xenial-osic-cloud1-s3500-8127579 cinder-api[8526]: DEBUG cinder.api.extensions [-] Loading extension cinder.api.contrib.standard_extensions from (pid=8526) load_extension /opt/stack/new/cinder/cinder/api/extensions.py:198 diff --git a/os_loganalyze/tests/test_filters.py b/os_loganalyze/tests/test_filters.py index 52c6105..7f8b973 100644 --- a/os_loganalyze/tests/test_filters.py +++ b/os_loganalyze/tests/test_filters.py @@ -36,6 +36,7 @@ class TestSupportsSevRegex(base.TestCase): yes("/n-api.txt.gz") yes("/c-vol.txt.gz") yes("/tempest.txt") + yes("/devstack@c-api.service.log.txt") # this specific bug was hit previously no("check/gate-horizon-python27/1dba20d/console.html") no("check/gate-tempest-dsvm-trove/c5950fc/console.html") @@ -66,6 +67,27 @@ class TestFilters(base.TestCase): self.assertIn("class='DEBUG", line) self.assertIn("href='#_2013-09-27_18_22_11_249'", line) + def test_systemd_filters(self): + gen = self.get_generator('devstack@c-api.service.log.txt') + # dump the header + gen.next() + # dump the systemd line + gen.next() + + # first line + line = gen.next() + self.assertIn("NONE", line) + self.assertIn("href='#_Mar_28_12_20_42_377230'", line) + + # second line + line = gen.next() + self.assertIn("WARNING", line) + self.assertIn("href='#_Mar_28_12_20_43_570064", line) + # third line + line = gen.next() + self.assertIn("DEBUG", line) + self.assertIn("href='#_Mar_28_12_20_44_172534'", line) + def test_devstack_filters(self): gen = self.get_generator('devstacklog.txt.gz') # dump the header