Merge "Add support for setting filters and views per file"

etc/os_loganalyze/wsgi.conf

@@ -1,6 +1,7 @@
 [general]
 filter = SevFilter
 view = HTMLView
+file_conditions = /etc/os-loganalyze/file_conditions.yaml
 
 [swift]
 authurl=https://keystone.example.org/v2.0/

os_loganalyze/filter.py

@@ -161,17 +161,26 @@ class NoFilter(object):
 def get_filter_generator(file_generator, environ, root_path, config):
     """Return the filter to use as per the config."""
 
+    # Check file specific conditions first
+    filter_selected = util.get_file_conditions('filter', file_generator,
+                                               environ, root_path, config)
+
+    # Otherwise use the defaults in the config
+    if not filter_selected:
+        if config.has_section('general'):
+            if config.has_option('general', 'filter'):
+                filter_selected = config.get('general', 'filter')
+
     minsev = util.parse_param(environ, 'level', default="NONE")
     limit = util.parse_param(environ, 'limit')
 
-    if config.has_section('general'):
-        if config.has_option('general', 'filter'):
-            set_filter = config.get('general', 'filter')
-            if set_filter.lower() in ['sevfilter', 'sev']:
-                return SevFilter(file_generator, minsev, limit)
-            elif set_filter.lower() in ['nofilter', 'no']:
-                return NoFilter(file_generator)
+    if filter_selected:
+        if filter_selected.lower() in ['sevfilter', 'sev']:
+            return SevFilter(file_generator, minsev, limit)
+        elif filter_selected.lower() in ['nofilter', 'no']:
+            return NoFilter(file_generator)
 
+    # Otherwise guess
     if util.use_passthrough_view(file_generator.file_headers):
         return NoFilter(file_generator)
 

os_loganalyze/tests/base.py

@@ -15,8 +15,10 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import ConfigParser
 import os
 import os.path
+import tempfile
 import urllib
 from wsgiref import util
 
@@ -80,6 +82,21 @@ class TestCase(testtools.TestCase):
         util.setup_testing_defaults(environ)
         return environ
 
+    def _create_wsgi_config_file_for_job(self):
+        # We need to create a new config file for each job run to have the
+        # opportunity to modify paths to the tests samples dir
+        config = ConfigParser.ConfigParser()
+        config.read(os.path.expanduser(self.wsgi_config_file))
+
+        if config.has_section('general'):
+            if config.has_option('general', 'file_conditions'):
+                config.set(
+                    'general', 'file_conditions',
+                    samples_path() + config.get('general', 'file_conditions'))
+        fd, filename = tempfile.mkstemp()
+        config.write(os.fdopen(fd, 'w'))
+        return filename
+
     def get_generator(self, fname, level=None, html=True,
                       limit=None, source=None):
         kwargs = {'PATH_INFO': '/htmlify/%s/%s' % (self.samples_directory,
@@ -101,6 +118,6 @@ class TestCase(testtools.TestCase):
             self.fake_env(**kwargs),
             self._start_response,
             root_path=samples_path(''),
-            wsgi_config=self.wsgi_config_file)
+            wsgi_config=self._create_wsgi_config_file_for_job())
 
         return iter(gen)

os_loganalyze/tests/samples/file_conditions.yaml

@@ -0,0 +1,10 @@
+conditions:
+ - filename_pattern: ^.*\.txt\.gz$
+   filter: SevFilter
+   view: HTMLView
+ - filename_pattern: ^.*\.txt?$
+   filter: SevFilter
+   view: TextView
+ - filename_pattern: ^.*$
+   filter: NoFilter
+   view: PassthroughView

os_loganalyze/tests/samples/simple.txt

@@ -0,0 +1,2 @@
+2013-09-27 18:22:35.392 testing 123
+2013-09-27 18:22:36.123 second line

os_loganalyze/tests/samples/wsgi_file_conditions.conf

@@ -0,0 +1,11 @@
+[general]
+file_conditions = file_conditions.yaml
+
+[swift]
+authurl=https://keystone.example.org/v2.0/
+user=example
+password=example
+container=logs
+region=EXP
+tenant=
+chunk_size=64

os_loganalyze/tests/test_wsgi.py

@@ -195,6 +195,28 @@ class TestWsgiDisk(base.TestCase):
         first = gen.next()
         self.assertNotIn('<html>', first)
 
+    def test_file_conditions(self):
+        self.wsgi_config_file = (base.samples_path('samples') +
+                                 'wsgi_file_conditions.conf')
+        # Check we are matching and setting the HTML filter
+        gen = self.get_generator('devstacklog.txt.gz')
+
+        first = gen.next()
+        self.assertIn('<html>', first)
+
+        # Check for simple.html we don't have HTML but do have date lines
+        gen = self.get_generator('simple.txt')
+
+        first = gen.next()
+        self.assertIn('2013-09-27 18:22:35.392 testing 123', first)
+
+        # Test images go through the passthrough filter
+        gen = self.get_generator('openstack_logo.png')
+        first = gen.next()
+        self.assertNotIn('html', first)
+        with open(base.samples_path('samples') + 'openstack_logo.png') as f:
+            self.assertEqual(first, f.readline())
+
 
 class TestWsgiSwift(TestWsgiDisk):
     """Test loading files from swift."""

os_loganalyze/util.py

@@ -15,10 +15,13 @@
 # under the License.
 
 import cgi
+import logging
 import os
+import re
 import time
 
 import magic
+import yaml
 
 
 def parse_param(env, name, default=None):
@@ -104,3 +107,31 @@ def use_passthrough_view(file_headers):
             if os.path.splitext(filename)[1] in ['.txt', '.html']:
                 return False
     return True
+
+
+def load_file_conditions(config):
+    if config.has_section('general'):
+        if config.has_option('general', 'file_conditions'):
+            try:
+                with open(config.get('general', 'file_conditions'), 'r') as f:
+                    fm = yaml.safe_load(f)
+                    return fm.get('conditions', [])
+            except Exception:
+                logging.warn("Failed to load file conditions")
+    return []
+
+
+def get_file_conditions(item, file_generator, environ, root_path, config):
+    """Get the matching item for the given file."""
+    # Also take in environ and root_path if in the future we want to match
+    # on other conditions
+
+    # We return the first match or None if nothing is found
+
+    conditions = load_file_conditions(config)
+    for cond in conditions:
+        if 'filename_pattern' in cond and item in cond:
+            if re.match(cond['filename_pattern'], file_generator.logname):
+                return cond[item]
+
+    return None

os_loganalyze/view.py

@@ -213,16 +213,26 @@ class PassthroughView(collections.Iterable):
 
 def get_view_generator(filter_generator, environ, root_path, config):
     """Return the view to use as per the config."""
-    if config.has_section('general'):
-        if config.has_option('general', 'view'):
-            set_view = config.get('general', 'view')
-            if set_view.lower() in ['htmlview', 'html']:
-                return HTMLView(filter_generator)
-            elif set_view.lower() in ['textview', 'text']:
-                return TextView(filter_generator)
-            elif set_view.lower() in ['passthroughview', 'passthrough']:
-                return PassthroughView(filter_generator)
+    # Check file specific conditions first
+    view_selected = util.get_file_conditions('view',
+                                             filter_generator.file_generator,
+                                             environ, root_path, config)
 
+    # Otherwise use the defaults in the config
+    if not view_selected:
+        if config.has_section('general'):
+            if config.has_option('general', 'view'):
+                view_selected = config.get('general', 'view')
+
+    if view_selected:
+        if view_selected.lower() in ['htmlview', 'html']:
+            return HTMLView(filter_generator)
+        elif view_selected.lower() in ['textview', 'text']:
+            return TextView(filter_generator)
+        elif view_selected.lower() in ['passthroughview', 'passthrough']:
+            return PassthroughView(filter_generator)
+
+    # Otherwise guess
     if util.use_passthrough_view(filter_generator.file_generator.file_headers):
         return PassthroughView(filter_generator)
     elif util.should_be_html(environ):

requirements.txt

@@ -3,3 +3,4 @@ Babel>=0.9.6
 python-swiftclient>=1.6
 python-keystoneclient>=0.4.2
 python-magic
+PyYAML