add utility to htmlify screen logs for devstack runs
this adds a wsgi application which processes all the txt.gz log files and htmlifies them on the fly before sending them to the user. It uses content negotiation to only do this if the client says it Accepts text/html, otherwise it just dumps them back out as text/plain. In either case the content is compressed with mod_deflate, which provides a 15x-20x bandwidth savings vs. if we left it uncompressed on the wire. Should get more testing before general deployment, but this is getting close to the approach I think we want to use. This enables the functionality on a new logs-dev.openstack.org vhost so that we can do live testing that this works before switching over the main site to it. Change-Id: I12a06c4b55be05fb8060b76f5aea9ebc801c0537
This commit is contained in:
Sean Dague
parent
368e7331cf
commit
fcc4dd4995
|
|
@ -0,0 +1,125 @@
|
|||
#!/usr/bin/python
|
||||
#
|
||||
# Copyright (c) 2013 IBM Corp.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
import cgi
|
||||
import fileinput
|
||||
import re
|
||||
import sys
|
||||
import wsgiref.util
|
||||
|
||||
|
||||
DATEFMT = '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3})?'
|
||||
STATUSFMT = '(DEBUG|INFO|WARN|ERROR|TRACE|AUDIT)'
|
||||
LOGMATCH = '(?P<date>%s)(?P<pid> \d+)? (?P<status>%s)' % (DATEFMT, STATUSFMT)
|
||||
|
||||
|
||||
def _html_close():
|
||||
return ("</pre></body></html>\n")
|
||||
|
||||
|
||||
def _css_preamble():
|
||||
"""Write a valid html start with css that we need."""
|
||||
return ("""<html>
|
||||
<head>
|
||||
<style>
|
||||
a {color: #000; text-decoration: none}
|
||||
a:hover {text-decoration: underline}
|
||||
.DEBUG, .DEBUG a {color: #888}
|
||||
.ERROR, .ERROR a {color: #c00; font-weight: bold}
|
||||
.TRACE, .TRACE a {color: #c60}
|
||||
.WARN, .WARN a {color: #D89100; font-weight: bold}
|
||||
.INFO, .INFO a {color: #006; font-weight: bold}
|
||||
</style>
|
||||
<body><pre>\n""")
|
||||
|
||||
|
||||
def color_by_sev(line):
|
||||
"""Wrap a line in a span whose class matches it's severity."""
|
||||
m = re.match(LOGMATCH, line)
|
||||
if m:
|
||||
return "<span class='%s'>%s</span>" % (m.group('status'), line)
|
||||
else:
|
||||
return line
|
||||
|
||||
|
||||
def escape_html(line):
|
||||
"""Escape the html in a line.
|
||||
|
||||
We need to do this because we dump xml into the logs, and if we don't
|
||||
escape the xml we end up with invisible parts of the logs in turning it
|
||||
into html.
|
||||
"""
|
||||
return cgi.escape(line)
|
||||
|
||||
|
||||
def link_timestamp(line):
|
||||
return re.sub('^(?P<span><span[^>]*>)?(?P<date>%s)' % DATEFMT,
|
||||
('\g<span><a name="\g<date>" class="date" '
|
||||
'href="#\g<date>">\g<date></a>'),
|
||||
line)
|
||||
|
||||
|
||||
def passthrough_filter(fname):
|
||||
for line in fileinput.input(fname, openhook=fileinput.hook_compressed):
|
||||
yield line
|
||||
|
||||
|
||||
def html_filter(fname):
|
||||
"""Generator to read logs and output html in a stream.
|
||||
|
||||
This produces a stream of the htmlified logs which lets us return
|
||||
data quickly to the user, and use minimal memory in the process.
|
||||
"""
|
||||
yield _css_preamble()
|
||||
for line in fileinput.input(fname, openhook=fileinput.hook_compressed):
|
||||
newline = escape_html(line)
|
||||
newline = color_by_sev(newline)
|
||||
newline = link_timestamp(newline)
|
||||
yield newline
|
||||
yield _html_close()
|
||||
|
||||
|
||||
def htmlify_stdin():
|
||||
out = sys.stdout
|
||||
out.write(_css_preamble())
|
||||
for line in fileinput.input():
|
||||
newline = escape_html(line)
|
||||
newline = color_by_sev(newline)
|
||||
newline = link_timestamp(newline)
|
||||
out.write(newline)
|
||||
out.write(_html_close())
|
||||
|
||||
|
||||
def application(environ, start_response):
|
||||
status = '200 OK'
|
||||
path = wsgiref.util.request_uri(environ)
|
||||
match = re.search('htmlify/(.*)', path)
|
||||
# TODO(sdague): scrub all .. chars out of the path, for security reasons
|
||||
fname = "/srv/static/logs/%s" % match.groups(1)[0]
|
||||
if 'HTTP_ACCEPT' in environ and 'text/html' in environ['HTTP_ACCEPT']:
|
||||
response_headers = [('Content-type', 'text/html')]
|
||||
start_response(status, response_headers)
|
||||
return html_filter(fname)
|
||||
else:
|
||||
response_headers = [('Content-type', 'text/plain')]
|
||||
start_response(status, response_headers)
|
||||
return passthrough_filter(fname)
|
||||
|
||||
|
||||
# for development purposes, makes it easy to test the filter output
|
||||
if __name__ == "__main__":
|
||||
htmlify_stdin()
|
||||
|
|
@ -21,6 +21,7 @@ class openstack_project::static (
|
|||
}
|
||||
|
||||
include apache
|
||||
include apache::mod::wsgi
|
||||
|
||||
a2mod { 'rewrite':
|
||||
ensure => present,
|
||||
|
|
@ -81,6 +82,14 @@ class openstack_project::static (
|
|||
template => 'openstack_project/logs.vhost.erb',
|
||||
}
|
||||
|
||||
apache::vhost { 'logs-dev.openstack.org':
|
||||
port => 80,
|
||||
priority => '51',
|
||||
docroot => '/srv/static/logs',
|
||||
require => File['/srv/static/logs'],
|
||||
template => 'openstack_project/logs-dev.vhost.erb',
|
||||
}
|
||||
|
||||
file { '/srv/static/logs':
|
||||
ensure => directory,
|
||||
owner => 'jenkins',
|
||||
|
|
@ -97,6 +106,14 @@ class openstack_project::static (
|
|||
require => File['/srv/static/logs'],
|
||||
}
|
||||
|
||||
file { '/usr/local/bin/htmlify-screen-log.py':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
source => 'puppet:///modules/openstack_project/logs/htmlify-screen-log.py',
|
||||
}
|
||||
|
||||
file { '/srv/static/logs/help':
|
||||
ensure => directory,
|
||||
recurse => true,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
# -*- apache -*-
|
||||
# ************************************
|
||||
# Managed by Puppet
|
||||
# ************************************
|
||||
|
||||
NameVirtualHost <%= vhost_name %>:<%= port %>
|
||||
<VirtualHost <%= vhost_name %>:<%= port %>>
|
||||
ServerName <%= srvname %>
|
||||
<% if serveraliases.is_a? Array -%>
|
||||
<% serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%>
|
||||
<% elsif serveraliases != '' -%>
|
||||
<%= " ServerAlias #{serveraliases}" %>
|
||||
<% end -%>
|
||||
DocumentRoot <%= docroot %>
|
||||
|
||||
RewriteEngine On
|
||||
# rewrite all txt.gz files to map to our internal htmlify wsgi app
|
||||
RewriteRule ^/(.*\.txt\.gz)$ /htmlify/$1 [QSA,L,PT]
|
||||
WSGIScriptAlias /htmlify /usr/local/bin/htmlify-screen-log.py
|
||||
# use Apache to compress the results afterwards, to save on the wire
|
||||
# it's approx 18x savings of wire traffic to compress. We need to
|
||||
# compress by content types that htmlify can produce
|
||||
AddOutputFilterByType DEFLATE text/plain text/html
|
||||
|
||||
<FilesMatch \.html\.gz$>
|
||||
ForceType text/html
|
||||
AddDefaultCharset UTF-8
|
||||
AddEncoding x-gzip gz
|
||||
</FilesMatch>
|
||||
<Directory <%= docroot %>>
|
||||
Options <%= options %>
|
||||
AllowOverride None
|
||||
Order allow,deny
|
||||
allow from all
|
||||
</Directory>
|
||||
<Directory <%= docroot %>/*/*/*/gate-tempest-devstack*/*>
|
||||
ReadmeName /help/tempest-overview.html
|
||||
</Directory>
|
||||
<Directory <%= docroot %>/*/*/*/gate-tempest-devstack*/*/logs/>
|
||||
ReadmeName /help/tempest-logs.html
|
||||
</Directory>
|
||||
|
||||
ErrorLog /var/log/apache2/<%= name %>_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/apache2/<%= name %>_access.log combined
|
||||
ServerSignature Off
|
||||
</VirtualHost>
|
||||
Loading…
Reference in New Issue