diff --git a/files/disallow_robots.txt b/files/disallow_robots.txt
new file mode 100644
index 0000000..1f53798
--- /dev/null
+++ b/files/disallow_robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
diff --git a/files/log_archive_maintenance.sh b/files/log_archive_maintenance.sh
new file mode 100644
index 0000000..71af22a
--- /dev/null
+++ b/files/log_archive_maintenance.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+sleep $((RANDOM%600)) && \
+flock -n /var/run/gziplogs.lock \
+find -O3 /srv/static/logs/ -depth -not -name robots.txt -not -name lost+found \
+ -not -wholename /srv/static/logs/help/\* \( \
+ \( -type f -mmin +10 -not -name \*\[.-\]gz -not -name \*\[._-\]\[zZ\] \
+ \( -name \*.txt -or -name \*.html -or -name tmp\* \) \
+ -exec gzip \{\} \; \) \
+ -o \( -type f -mtime +120 -execdir rm \{\} \; \) \
+ -o \( -type d -empty -mtime +1 -execdir rmdir {} \; \) \)
+find -O3 /srv/static/docs-draft/ -depth -not -name lost+found \( \
+ \( -type f -mtime +21 -execdir rm \{\} \; \) \
+ -o \( -type d -empty -mtime +1 -execdir rmdir {} \; \) \)
diff --git a/manifests/logserver.pp b/manifests/logserver.pp
new file mode 100644
index 0000000..245fe2d
--- /dev/null
+++ b/manifests/logserver.pp
@@ -0,0 +1,183 @@
+# Copyright (c) 2012-2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# == Class: openstackci::logserver
+#
+class openstackci::logserver (
+ $domain,
+ $jenkins_ssh_key,
+ $swift_authurl = '',
+ $swift_user = '',
+ $swift_key = '',
+ $swift_tenant_name = '',
+ $swift_region_name = '',
+ $swift_default_container = '',
+) {
+
+ if ! defined(Class['::jenkins::jenkinsuser']) {
+ class { '::jenkins::jenkinsuser':
+ ssh_key => $jenkins_ssh_key,
+ }
+ }
+
+ include apache
+ include apache::mod::wsgi
+
+ if ! defined(A2mod['rewrite']) {
+ a2mod { 'rewrite':
+ ensure => present,
+ }
+ }
+
+ if ! defined(A2mod['proxy']) {
+ a2mod { 'proxy':
+ ensure => present,
+ }
+ }
+
+ if ! defined(A2mod['proxy_http']) {
+ a2mod { 'proxy_http':
+ ensure => present,
+ }
+ }
+
+ if ! defined(File['/srv/static']) {
+ file { '/srv/static':
+ ensure => directory,
+ }
+ }
+
+ apache::vhost { "logs.${domain}":
+ port => 80,
+ priority => '50',
+ docroot => '/srv/static/logs',
+ require => File['/srv/static/logs'],
+ template => 'openstackci/logs.vhost.erb',
+ }
+
+ apache::vhost { "logs-dev.${domain}":
+ port => 80,
+ priority => '51',
+ docroot => '/srv/static/logs',
+ require => File['/srv/static/logs'],
+ template => 'openstackci/logs-dev.vhost.erb',
+ }
+
+ file { '/srv/static/logs':
+ ensure => directory,
+ owner => 'jenkins',
+ group => 'jenkins',
+ require => User['jenkins'],
+ }
+
+ file { '/srv/static/logs/robots.txt':
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ source => 'puppet:///modules/openstackci/disallow_robots.txt',
+ require => File['/srv/static/logs'],
+ }
+
+ package { 'keyring':
+ ensure => 'latest',
+ provider => 'pip',
+ }
+
+ vcsrepo { '/opt/os-loganalyze':
+ ensure => latest,
+ provider => git,
+ revision => 'master',
+ source => 'https://git.openstack.org/openstack-infra/os-loganalyze',
+ require => Package['keyring'],
+ }
+
+ exec { 'install_os-loganalyze':
+ command => 'pip install .',
+ cwd => '/opt/os-loganalyze',
+ path => '/usr/local/bin:/usr/bin:/bin/',
+ refreshonly => true,
+ subscribe => Vcsrepo['/opt/os-loganalyze'],
+ }
+
+ file { '/etc/os_loganalyze':
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ require => Vcsrepo['/opt/os-loganalyze'],
+ }
+
+ file { '/etc/os_loganalyze/wsgi.conf':
+ ensure => present,
+ owner => 'root',
+ group => 'www-data',
+ mode => '0440',
+ content => template('openstackci/os-loganalyze-wsgi.conf.erb'),
+ require => File['/etc/os_loganalyze'],
+ }
+
+ vcsrepo { '/opt/devstack-gate':
+ ensure => latest,
+ provider => git,
+ revision => 'master',
+ source => 'https://git.openstack.org/openstack-infra/devstack-gate',
+ }
+
+ file { '/srv/static/logs/help':
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ require => File['/srv/static/logs'],
+ }
+
+ file { '/srv/static/logs/help/tempest-logs.html':
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ source => 'file:///opt/devstack-gate/help/tempest-logs.html',
+ require => [File['/srv/static/logs/help'], Vcsrepo['/opt/devstack-gate']],
+ }
+
+ file { '/srv/static/logs/help/tempest-overview.html':
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ source => 'file:///opt/devstack-gate/help/tempest-overview.html',
+ require => [File['/srv/static/logs/help'], Vcsrepo['/opt/devstack-gate']],
+ }
+
+ file { '/usr/local/sbin/log_archive_maintenance.sh':
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0744',
+ source => 'puppet:///modules/openstackci/log_archive_maintenance.sh',
+ }
+
+ cron { 'gziprmlogs':
+ user => 'root',
+ minute => '0',
+ hour => '7',
+ weekday => '6',
+ command => 'bash /usr/local/sbin/log_archive_maintenance.sh',
+ environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin',
+ require => File['/usr/local/sbin/log_archive_maintenance.sh'],
+ }
+
+}
diff --git a/metadata.json b/metadata.json
index b376fda..8d414e1 100644
--- a/metadata.json
+++ b/metadata.json
@@ -8,5 +8,8 @@
"project_page": "http://ci.openstack.org/",
"issues_url": "https://storyboard.openstack.org/#!/project/808",
"dependencies": [
+ { "name": "puppetlabs/apache" },
+ { "name": "openstackinfra/vcsrepo" },
+ { "name": "openstackinfra/jenkins" }
]
}
diff --git a/templates/logs-dev.vhost.erb b/templates/logs-dev.vhost.erb
new file mode 100644
index 0000000..48d6e1b
--- /dev/null
+++ b/templates/logs-dev.vhost.erb
@@ -0,0 +1,88 @@
+# -*- apache -*-
+# ************************************
+# Managed by Puppet
+# ************************************
+
+NameVirtualHost <%= @vhost_name %>:<%= @port %>
+:<%= @port %>>
+ ServerName <%= @srvname %>
+<% if @serveraliases.is_a? Array -%>
+<% @serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%>
+<% elsif @serveraliases != '' -%>
+<%= " ServerAlias #{@serveraliases}" %>
+<% end -%>
+ DocumentRoot <%= @docroot %>
+
+ # use Apache to compress the results afterwards, to save on the wire
+ # it's approx 18x savings of wire traffic to compress. We need to
+ # compress by content types that htmlify can produce
+ AddOutputFilterByType DEFLATE text/plain text/html
+
+
+ ForceType text/html
+ AddDefaultCharset UTF-8
+ AddEncoding x-gzip gz
+
+ >
+ Options <%= @options %>
+ AllowOverride None
+ Order allow,deny
+ allow from all
+ Satisfy Any
+
+
+ Allow from all
+ Satisfy Any
+
+
+ /*/*/*/*/*-tempest-dsvm*/*>
+ ReadmeName /help/tempest-overview.html
+
+ /periodic*/*/*-tempest-dsvm*/*>
+ ReadmeName /help/tempest-overview.html
+
+ /*/*/*/*/*-tempest-dsvm*/*/logs/>
+ ReadmeName /help/tempest-logs.html
+
+ /periodic*/*/*-tempest-dsvm*/*/logs/>
+ ReadmeName /help/tempest-logs.html
+
+
+ # old patterns, can be removed in July 2014
+ /*/*/*/*/gate-tempest-devstack*/*>
+ ReadmeName /help/tempest-overview.html
+
+ /periodic/*/periodic-tempest-devstack*/*>
+ ReadmeName /help/tempest-overview.html
+
+ /*/*/*/*/gate-tempest-devstack*/*/logs/>
+ ReadmeName /help/tempest-logs.html
+
+ /periodic/*/periodic-tempest-devstack*/*/logs/>
+ ReadmeName /help/tempest-logs.html
+
+
+ RewriteEngine On
+ # rewrite all txt.gz & html.gz files to map to our internal htmlify wsgi app
+ # PT, Pass-through: to come back around and get picked up by the WSGIScriptAlias
+ # NS, No-subrequest: on coming back through, mod-autoindex may have added index.html
+ # which would match the !-f condition. We therefore ensure the
+ # rewrite doesn't trigger by disallowing subrequests.
+ RewriteRule ^/(.*\.txt\.gz)$ /htmlify/$1 [QSA,L,PT,NS]
+ RewriteRule ^/(.*console\.html(\.gz)?)$ /htmlify/$1 [QSA,L,PT,NS]
+
+ # Check if the request exists as a file, directory or symbolic link
+ # If not, write the request to htmlify to see if we can fetch from swift
+ RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-f
+ RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-d
+ RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-l
+ RewriteCond %{REQUEST_FILENAME} !^/icon
+ RewriteRule ^/(.*)$ /htmlify/$1 [QSA,L,PT,NS]
+
+ WSGIScriptAlias /htmlify /usr/local/lib/python2.7/dist-packages/os_loganalyze/wsgi.py
+
+ ErrorLog /var/log/apache2/<%= @name %>_error.log
+ LogLevel warn
+ CustomLog /var/log/apache2/<%= @name %>_access.log combined
+ ServerSignature Off
+
diff --git a/templates/logs.vhost.erb b/templates/logs.vhost.erb
new file mode 100644
index 0000000..1bec5be
--- /dev/null
+++ b/templates/logs.vhost.erb
@@ -0,0 +1,91 @@
+# -*- apache -*-
+# ************************************
+# Managed by Puppet
+# ************************************
+
+NameVirtualHost <%= vhost_name %>:<%= port %>
+:<%= port %>>
+ ServerName <%= srvname %>
+<% if serveraliases.is_a? Array -%>
+<% serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%>
+<% elsif serveraliases != '' -%>
+<%= " ServerAlias #{serveraliases}" %>
+<% end -%>
+ DocumentRoot <%= docroot %>
+
+ # use Apache to compress the results afterwards, to save on the wire
+ # it's approx 18x savings of wire traffic to compress. We need to
+ # compress by content types that htmlify can produce
+ AddOutputFilterByType DEFLATE text/plain text/html
+
+
+ ForceType text/html
+ AddDefaultCharset UTF-8
+ AddEncoding x-gzip gz
+
+ >
+ Options <%= options %>
+ AllowOverride None
+ Order allow,deny
+ allow from all
+ Satisfy Any
+
+
+ Allow from all
+ Satisfy Any
+
+
+ /*/*/*/*/*-tempest-dsvm*/*>
+ ReadmeName /help/tempest-overview.html
+
+ /periodic*/*/*-tempest-dsvm*/*>
+ ReadmeName /help/tempest-overview.html
+
+ /*/*/*/*/*-tempest-dsvm*/*/logs/>
+ ReadmeName /help/tempest-logs.html
+
+ /periodic*/*/*-tempest-dsvm*/*/logs/>
+ ReadmeName /help/tempest-logs.html
+
+
+ # old patterns, can be removed in July 2014
+ /*/*/*/*/gate-tempest-devstack*/*>
+ ReadmeName /help/tempest-overview.html
+
+ /periodic/*/periodic-tempest-devstack*/*>
+ ReadmeName /help/tempest-overview.html
+
+ /*/*/*/*/gate-tempest-devstack*/*/logs/>
+ ReadmeName /help/tempest-logs.html
+
+ /periodic/*/periodic-tempest-devstack*/*/logs/>
+ ReadmeName /help/tempest-logs.html
+
+
+ RewriteEngine On
+ # rewrite txt.gz & console.html[.gz] files to map to our internal htmlify
+ # wsgi app
+ # PT, Pass-through: to come back around and get picked up by the
+ # WSGIScriptAlias
+ # NS, No-subrequest: on coming back through, mod-autoindex may have added
+ # index.html which would match the !-f condition. We
+ # therefore ensure the rewrite doesn't trigger by
+ # disallowing subrequests.
+ RewriteRule ^/(.*\.txt\.gz)$ /htmlify/$1 [QSA,L,PT,NS]
+ RewriteRule ^/(.*console\.html(\.gz)?)$ /htmlify/$1 [QSA,L,PT,NS]
+
+ # Check if the request exists as a file, directory or symbolic link
+ # If not, write the request to htmlify to see if we can fetch from swift
+ RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-f
+ RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-d
+ RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-l
+ RewriteCond %{REQUEST_FILENAME} !^/icon
+ RewriteRule ^/(.*)$ /htmlify/$1 [QSA,L,PT,NS]
+
+ WSGIScriptAlias /htmlify /usr/local/lib/python2.7/dist-packages/os_loganalyze/wsgi.py
+
+ ErrorLog /var/log/apache2/<%= name %>_error.log
+ LogLevel warn
+ CustomLog /var/log/apache2/<%= name %>_access.log combined
+ ServerSignature Off
+
diff --git a/templates/os-loganalyze-wsgi.conf.erb b/templates/os-loganalyze-wsgi.conf.erb
new file mode 100644
index 0000000..a917670
--- /dev/null
+++ b/templates/os-loganalyze-wsgi.conf.erb
@@ -0,0 +1,10 @@
+<% if swift_authurl != "" -%>
+[swift]
+authurl=<%= swift_authurl %>
+user=<%= swift_user %>
+password=<%= swift_key %>
+tenant=<%= swift_tenant_name %>
+region=<%= swift_region_name %>
+chunk_size=64
+container=<%= swift_default_container %>
+<% end -%>