Add support for setting filters and views per file
Match filters and views to files based off their filename. If no match for a file is found the default set in wsgi.conf will be used or else a best (legacy) guess is made. Change-Id: Idac2849b52471acb06eb43b64edb1762a74c6fe1
This commit is contained in:
parent
d4ac63b711
commit
2dac23f208
|
@ -1,6 +1,7 @@
|
|||
[general]
|
||||
filter = SevFilter
|
||||
view = HTMLView
|
||||
file_conditions = /etc/os-loganalyze/file_conditions.yaml
|
||||
|
||||
[swift]
|
||||
authurl=https://keystone.example.org/v2.0/
|
||||
|
|
|
@ -161,17 +161,26 @@ class NoFilter(object):
|
|||
def get_filter_generator(file_generator, environ, root_path, config):
|
||||
"""Return the filter to use as per the config."""
|
||||
|
||||
# Check file specific conditions first
|
||||
filter_selected = util.get_file_conditions('filter', file_generator,
|
||||
environ, root_path, config)
|
||||
|
||||
# Otherwise use the defaults in the config
|
||||
if not filter_selected:
|
||||
if config.has_section('general'):
|
||||
if config.has_option('general', 'filter'):
|
||||
filter_selected = config.get('general', 'filter')
|
||||
|
||||
minsev = util.parse_param(environ, 'level', default="NONE")
|
||||
limit = util.parse_param(environ, 'limit')
|
||||
|
||||
if config.has_section('general'):
|
||||
if config.has_option('general', 'filter'):
|
||||
set_filter = config.get('general', 'filter')
|
||||
if set_filter.lower() in ['sevfilter', 'sev']:
|
||||
return SevFilter(file_generator, minsev, limit)
|
||||
elif set_filter.lower() in ['nofilter', 'no']:
|
||||
return NoFilter(file_generator)
|
||||
if filter_selected:
|
||||
if filter_selected.lower() in ['sevfilter', 'sev']:
|
||||
return SevFilter(file_generator, minsev, limit)
|
||||
elif filter_selected.lower() in ['nofilter', 'no']:
|
||||
return NoFilter(file_generator)
|
||||
|
||||
# Otherwise guess
|
||||
if util.use_passthrough_view(file_generator.file_headers):
|
||||
return NoFilter(file_generator)
|
||||
|
||||
|
|
|
@ -15,8 +15,10 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import ConfigParser
|
||||
import os
|
||||
import os.path
|
||||
import tempfile
|
||||
import urllib
|
||||
from wsgiref import util
|
||||
|
||||
|
@ -80,6 +82,21 @@ class TestCase(testtools.TestCase):
|
|||
util.setup_testing_defaults(environ)
|
||||
return environ
|
||||
|
||||
def _create_wsgi_config_file_for_job(self):
|
||||
# We need to create a new config file for each job run to have the
|
||||
# opportunity to modify paths to the tests samples dir
|
||||
config = ConfigParser.ConfigParser()
|
||||
config.read(os.path.expanduser(self.wsgi_config_file))
|
||||
|
||||
if config.has_section('general'):
|
||||
if config.has_option('general', 'file_conditions'):
|
||||
config.set(
|
||||
'general', 'file_conditions',
|
||||
samples_path() + config.get('general', 'file_conditions'))
|
||||
fd, filename = tempfile.mkstemp()
|
||||
config.write(os.fdopen(fd, 'w'))
|
||||
return filename
|
||||
|
||||
def get_generator(self, fname, level=None, html=True,
|
||||
limit=None, source=None):
|
||||
kwargs = {'PATH_INFO': '/htmlify/%s/%s' % (self.samples_directory,
|
||||
|
@ -101,6 +118,6 @@ class TestCase(testtools.TestCase):
|
|||
self.fake_env(**kwargs),
|
||||
self._start_response,
|
||||
root_path=samples_path(''),
|
||||
wsgi_config=self.wsgi_config_file)
|
||||
wsgi_config=self._create_wsgi_config_file_for_job())
|
||||
|
||||
return iter(gen)
|
||||
|
|
|
@ -0,0 +1,10 @@
|
|||
conditions:
|
||||
- filename_pattern: ^.*\.txt\.gz$
|
||||
filter: SevFilter
|
||||
view: HTMLView
|
||||
- filename_pattern: ^.*\.txt?$
|
||||
filter: SevFilter
|
||||
view: TextView
|
||||
- filename_pattern: ^.*$
|
||||
filter: NoFilter
|
||||
view: PassthroughView
|
|
@ -0,0 +1,2 @@
|
|||
2013-09-27 18:22:35.392 testing 123
|
||||
2013-09-27 18:22:36.123 second line
|
|
@ -0,0 +1,11 @@
|
|||
[general]
|
||||
file_conditions = file_conditions.yaml
|
||||
|
||||
[swift]
|
||||
authurl=https://keystone.example.org/v2.0/
|
||||
user=example
|
||||
password=example
|
||||
container=logs
|
||||
region=EXP
|
||||
tenant=
|
||||
chunk_size=64
|
|
@ -195,6 +195,28 @@ class TestWsgiDisk(base.TestCase):
|
|||
first = gen.next()
|
||||
self.assertNotIn('<html>', first)
|
||||
|
||||
def test_file_conditions(self):
|
||||
self.wsgi_config_file = (base.samples_path('samples') +
|
||||
'wsgi_file_conditions.conf')
|
||||
# Check we are matching and setting the HTML filter
|
||||
gen = self.get_generator('devstacklog.txt.gz')
|
||||
|
||||
first = gen.next()
|
||||
self.assertIn('<html>', first)
|
||||
|
||||
# Check for simple.html we don't have HTML but do have date lines
|
||||
gen = self.get_generator('simple.txt')
|
||||
|
||||
first = gen.next()
|
||||
self.assertIn('2013-09-27 18:22:35.392 testing 123', first)
|
||||
|
||||
# Test images go through the passthrough filter
|
||||
gen = self.get_generator('openstack_logo.png')
|
||||
first = gen.next()
|
||||
self.assertNotIn('html', first)
|
||||
with open(base.samples_path('samples') + 'openstack_logo.png') as f:
|
||||
self.assertEqual(first, f.readline())
|
||||
|
||||
|
||||
class TestWsgiSwift(TestWsgiDisk):
|
||||
"""Test loading files from swift."""
|
||||
|
|
|
@ -15,10 +15,13 @@
|
|||
# under the License.
|
||||
|
||||
import cgi
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
import magic
|
||||
import yaml
|
||||
|
||||
|
||||
def parse_param(env, name, default=None):
|
||||
|
@ -104,3 +107,31 @@ def use_passthrough_view(file_headers):
|
|||
if os.path.splitext(filename)[1] in ['.txt', '.html']:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def load_file_conditions(config):
|
||||
if config.has_section('general'):
|
||||
if config.has_option('general', 'file_conditions'):
|
||||
try:
|
||||
with open(config.get('general', 'file_conditions'), 'r') as f:
|
||||
fm = yaml.safe_load(f)
|
||||
return fm.get('conditions', [])
|
||||
except Exception:
|
||||
logging.warn("Failed to load file conditions")
|
||||
return []
|
||||
|
||||
|
||||
def get_file_conditions(item, file_generator, environ, root_path, config):
|
||||
"""Get the matching item for the given file."""
|
||||
# Also take in environ and root_path if in the future we want to match
|
||||
# on other conditions
|
||||
|
||||
# We return the first match or None if nothing is found
|
||||
|
||||
conditions = load_file_conditions(config)
|
||||
for cond in conditions:
|
||||
if 'filename_pattern' in cond and item in cond:
|
||||
if re.match(cond['filename_pattern'], file_generator.logname):
|
||||
return cond[item]
|
||||
|
||||
return None
|
||||
|
|
|
@ -213,16 +213,26 @@ class PassthroughView(collections.Iterable):
|
|||
|
||||
def get_view_generator(filter_generator, environ, root_path, config):
|
||||
"""Return the view to use as per the config."""
|
||||
if config.has_section('general'):
|
||||
if config.has_option('general', 'view'):
|
||||
set_view = config.get('general', 'view')
|
||||
if set_view.lower() in ['htmlview', 'html']:
|
||||
return HTMLView(filter_generator)
|
||||
elif set_view.lower() in ['textview', 'text']:
|
||||
return TextView(filter_generator)
|
||||
elif set_view.lower() in ['passthroughview', 'passthrough']:
|
||||
return PassthroughView(filter_generator)
|
||||
# Check file specific conditions first
|
||||
view_selected = util.get_file_conditions('view',
|
||||
filter_generator.file_generator,
|
||||
environ, root_path, config)
|
||||
|
||||
# Otherwise use the defaults in the config
|
||||
if not view_selected:
|
||||
if config.has_section('general'):
|
||||
if config.has_option('general', 'view'):
|
||||
view_selected = config.get('general', 'view')
|
||||
|
||||
if view_selected:
|
||||
if view_selected.lower() in ['htmlview', 'html']:
|
||||
return HTMLView(filter_generator)
|
||||
elif view_selected.lower() in ['textview', 'text']:
|
||||
return TextView(filter_generator)
|
||||
elif view_selected.lower() in ['passthroughview', 'passthrough']:
|
||||
return PassthroughView(filter_generator)
|
||||
|
||||
# Otherwise guess
|
||||
if util.use_passthrough_view(filter_generator.file_generator.file_headers):
|
||||
return PassthroughView(filter_generator)
|
||||
elif util.should_be_html(environ):
|
||||
|
|
|
@ -3,3 +3,4 @@ Babel>=0.9.6
|
|||
python-swiftclient>=1.6
|
||||
python-keystoneclient>=0.4.2
|
||||
python-magic
|
||||
PyYAML
|
||||
|
|
Loading…
Reference in New Issue