diff --git a/etc/os_loganalyze/wsgi.conf b/etc/os_loganalyze/wsgi.conf index 7c0eeac..c2f7c5b 100644 --- a/etc/os_loganalyze/wsgi.conf +++ b/etc/os_loganalyze/wsgi.conf @@ -1,6 +1,7 @@ [general] filter = SevFilter view = HTMLView +file_conditions = /etc/os-loganalyze/file_conditions.yaml [swift] authurl=https://keystone.example.org/v2.0/ diff --git a/os_loganalyze/filter.py b/os_loganalyze/filter.py index 6b13a98..9ed6aa0 100644 --- a/os_loganalyze/filter.py +++ b/os_loganalyze/filter.py @@ -161,17 +161,26 @@ class NoFilter(object): def get_filter_generator(file_generator, environ, root_path, config): """Return the filter to use as per the config.""" + # Check file specific conditions first + filter_selected = util.get_file_conditions('filter', file_generator, + environ, root_path, config) + + # Otherwise use the defaults in the config + if not filter_selected: + if config.has_section('general'): + if config.has_option('general', 'filter'): + filter_selected = config.get('general', 'filter') + minsev = util.parse_param(environ, 'level', default="NONE") limit = util.parse_param(environ, 'limit') - if config.has_section('general'): - if config.has_option('general', 'filter'): - set_filter = config.get('general', 'filter') - if set_filter.lower() in ['sevfilter', 'sev']: - return SevFilter(file_generator, minsev, limit) - elif set_filter.lower() in ['nofilter', 'no']: - return NoFilter(file_generator) + if filter_selected: + if filter_selected.lower() in ['sevfilter', 'sev']: + return SevFilter(file_generator, minsev, limit) + elif filter_selected.lower() in ['nofilter', 'no']: + return NoFilter(file_generator) + # Otherwise guess if util.use_passthrough_view(file_generator.file_headers): return NoFilter(file_generator) diff --git a/os_loganalyze/tests/base.py b/os_loganalyze/tests/base.py index 8f4373f..321fc90 100644 --- a/os_loganalyze/tests/base.py +++ b/os_loganalyze/tests/base.py @@ -15,8 +15,10 @@ # License for the specific language governing permissions and limitations # under the License. +import ConfigParser import os import os.path +import tempfile import urllib from wsgiref import util @@ -80,6 +82,21 @@ class TestCase(testtools.TestCase): util.setup_testing_defaults(environ) return environ + def _create_wsgi_config_file_for_job(self): + # We need to create a new config file for each job run to have the + # opportunity to modify paths to the tests samples dir + config = ConfigParser.ConfigParser() + config.read(os.path.expanduser(self.wsgi_config_file)) + + if config.has_section('general'): + if config.has_option('general', 'file_conditions'): + config.set( + 'general', 'file_conditions', + samples_path() + config.get('general', 'file_conditions')) + fd, filename = tempfile.mkstemp() + config.write(os.fdopen(fd, 'w')) + return filename + def get_generator(self, fname, level=None, html=True, limit=None, source=None): kwargs = {'PATH_INFO': '/htmlify/%s/%s' % (self.samples_directory, @@ -101,6 +118,6 @@ class TestCase(testtools.TestCase): self.fake_env(**kwargs), self._start_response, root_path=samples_path(''), - wsgi_config=self.wsgi_config_file) + wsgi_config=self._create_wsgi_config_file_for_job()) return iter(gen) diff --git a/os_loganalyze/tests/samples/file_conditions.yaml b/os_loganalyze/tests/samples/file_conditions.yaml new file mode 100644 index 0000000..4ce8104 --- /dev/null +++ b/os_loganalyze/tests/samples/file_conditions.yaml @@ -0,0 +1,10 @@ +conditions: + - filename_pattern: ^.*\.txt\.gz$ + filter: SevFilter + view: HTMLView + - filename_pattern: ^.*\.txt?$ + filter: SevFilter + view: TextView + - filename_pattern: ^.*$ + filter: NoFilter + view: PassthroughView diff --git a/os_loganalyze/tests/samples/simple.txt b/os_loganalyze/tests/samples/simple.txt new file mode 100644 index 0000000..05b19b7 --- /dev/null +++ b/os_loganalyze/tests/samples/simple.txt @@ -0,0 +1,2 @@ +2013-09-27 18:22:35.392 testing 123 +2013-09-27 18:22:36.123 second line diff --git a/os_loganalyze/tests/samples/wsgi_file_conditions.conf b/os_loganalyze/tests/samples/wsgi_file_conditions.conf new file mode 100644 index 0000000..795b073 --- /dev/null +++ b/os_loganalyze/tests/samples/wsgi_file_conditions.conf @@ -0,0 +1,11 @@ +[general] +file_conditions = file_conditions.yaml + +[swift] +authurl=https://keystone.example.org/v2.0/ +user=example +password=example +container=logs +region=EXP +tenant= +chunk_size=64 diff --git a/os_loganalyze/tests/test_wsgi.py b/os_loganalyze/tests/test_wsgi.py index 99079aa..b8a48bf 100644 --- a/os_loganalyze/tests/test_wsgi.py +++ b/os_loganalyze/tests/test_wsgi.py @@ -195,6 +195,28 @@ class TestWsgiDisk(base.TestCase): first = gen.next() self.assertNotIn('', first) + def test_file_conditions(self): + self.wsgi_config_file = (base.samples_path('samples') + + 'wsgi_file_conditions.conf') + # Check we are matching and setting the HTML filter + gen = self.get_generator('devstacklog.txt.gz') + + first = gen.next() + self.assertIn('', first) + + # Check for simple.html we don't have HTML but do have date lines + gen = self.get_generator('simple.txt') + + first = gen.next() + self.assertIn('2013-09-27 18:22:35.392 testing 123', first) + + # Test images go through the passthrough filter + gen = self.get_generator('openstack_logo.png') + first = gen.next() + self.assertNotIn('html', first) + with open(base.samples_path('samples') + 'openstack_logo.png') as f: + self.assertEqual(first, f.readline()) + class TestWsgiSwift(TestWsgiDisk): """Test loading files from swift.""" diff --git a/os_loganalyze/util.py b/os_loganalyze/util.py index 87e5be9..d60433b 100644 --- a/os_loganalyze/util.py +++ b/os_loganalyze/util.py @@ -15,10 +15,13 @@ # under the License. import cgi +import logging import os +import re import time import magic +import yaml def parse_param(env, name, default=None): @@ -104,3 +107,31 @@ def use_passthrough_view(file_headers): if os.path.splitext(filename)[1] in ['.txt', '.html']: return False return True + + +def load_file_conditions(config): + if config.has_section('general'): + if config.has_option('general', 'file_conditions'): + try: + with open(config.get('general', 'file_conditions'), 'r') as f: + fm = yaml.safe_load(f) + return fm.get('conditions', []) + except Exception: + logging.warn("Failed to load file conditions") + return [] + + +def get_file_conditions(item, file_generator, environ, root_path, config): + """Get the matching item for the given file.""" + # Also take in environ and root_path if in the future we want to match + # on other conditions + + # We return the first match or None if nothing is found + + conditions = load_file_conditions(config) + for cond in conditions: + if 'filename_pattern' in cond and item in cond: + if re.match(cond['filename_pattern'], file_generator.logname): + return cond[item] + + return None diff --git a/os_loganalyze/view.py b/os_loganalyze/view.py index 91a8e18..b886f02 100644 --- a/os_loganalyze/view.py +++ b/os_loganalyze/view.py @@ -213,16 +213,26 @@ class PassthroughView(collections.Iterable): def get_view_generator(filter_generator, environ, root_path, config): """Return the view to use as per the config.""" - if config.has_section('general'): - if config.has_option('general', 'view'): - set_view = config.get('general', 'view') - if set_view.lower() in ['htmlview', 'html']: - return HTMLView(filter_generator) - elif set_view.lower() in ['textview', 'text']: - return TextView(filter_generator) - elif set_view.lower() in ['passthroughview', 'passthrough']: - return PassthroughView(filter_generator) + # Check file specific conditions first + view_selected = util.get_file_conditions('view', + filter_generator.file_generator, + environ, root_path, config) + # Otherwise use the defaults in the config + if not view_selected: + if config.has_section('general'): + if config.has_option('general', 'view'): + view_selected = config.get('general', 'view') + + if view_selected: + if view_selected.lower() in ['htmlview', 'html']: + return HTMLView(filter_generator) + elif view_selected.lower() in ['textview', 'text']: + return TextView(filter_generator) + elif view_selected.lower() in ['passthroughview', 'passthrough']: + return PassthroughView(filter_generator) + + # Otherwise guess if util.use_passthrough_view(filter_generator.file_generator.file_headers): return PassthroughView(filter_generator) elif util.should_be_html(environ): diff --git a/requirements.txt b/requirements.txt index 86870e4..be7dace 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ Babel>=0.9.6 python-swiftclient>=1.6 python-keystoneclient>=0.4.2 python-magic +PyYAML