Create a log server like http://logs.openstack.org
Copy the log server templates & files used in: https://git.openstack.org/cgit/openstack-infra/system-config/tree/modules/openstack_project Refactor the log server portion out of: https://git.openstack.org/cgit/openstack-infra/system-config/tree/modules/openstack_project/manifests/static.pp Additional modifications: 1. Add protections for portions that are likely to conflict. 2. Update os_loganalyzer install to use pip instead of python Related system-config change: I390f6ea8aaace276d211d55a0e17f25dd6ae26b5 Change-Id: Ib8476df83b7c5491158fa3cab588213be60fa2ab
This commit is contained in:
parent
f822432cb1
commit
e8106a86e6
|
@ -0,0 +1,2 @@
|
||||||
|
User-agent: *
|
||||||
|
Disallow: /
|
|
@ -0,0 +1,13 @@
|
||||||
|
#!/bin/sh
|
||||||
|
sleep $((RANDOM%600)) && \
|
||||||
|
flock -n /var/run/gziplogs.lock \
|
||||||
|
find -O3 /srv/static/logs/ -depth -not -name robots.txt -not -name lost+found \
|
||||||
|
-not -wholename /srv/static/logs/help/\* \( \
|
||||||
|
\( -type f -mmin +10 -not -name \*\[.-\]gz -not -name \*\[._-\]\[zZ\] \
|
||||||
|
\( -name \*.txt -or -name \*.html -or -name tmp\* \) \
|
||||||
|
-exec gzip \{\} \; \) \
|
||||||
|
-o \( -type f -mtime +120 -execdir rm \{\} \; \) \
|
||||||
|
-o \( -type d -empty -mtime +1 -execdir rmdir {} \; \) \)
|
||||||
|
find -O3 /srv/static/docs-draft/ -depth -not -name lost+found \( \
|
||||||
|
\( -type f -mtime +21 -execdir rm \{\} \; \) \
|
||||||
|
-o \( -type d -empty -mtime +1 -execdir rmdir {} \; \) \)
|
|
@ -0,0 +1,183 @@
|
||||||
|
# Copyright (c) 2012-2015 Hewlett-Packard Development Company, L.P.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# == Class: openstackci::logserver
|
||||||
|
#
|
||||||
|
class openstackci::logserver (
|
||||||
|
$domain,
|
||||||
|
$jenkins_ssh_key,
|
||||||
|
$swift_authurl = '',
|
||||||
|
$swift_user = '',
|
||||||
|
$swift_key = '',
|
||||||
|
$swift_tenant_name = '',
|
||||||
|
$swift_region_name = '',
|
||||||
|
$swift_default_container = '',
|
||||||
|
) {
|
||||||
|
|
||||||
|
if ! defined(Class['::jenkins::jenkinsuser']) {
|
||||||
|
class { '::jenkins::jenkinsuser':
|
||||||
|
ssh_key => $jenkins_ssh_key,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
include apache
|
||||||
|
include apache::mod::wsgi
|
||||||
|
|
||||||
|
if ! defined(A2mod['rewrite']) {
|
||||||
|
a2mod { 'rewrite':
|
||||||
|
ensure => present,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ! defined(A2mod['proxy']) {
|
||||||
|
a2mod { 'proxy':
|
||||||
|
ensure => present,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ! defined(A2mod['proxy_http']) {
|
||||||
|
a2mod { 'proxy_http':
|
||||||
|
ensure => present,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ! defined(File['/srv/static']) {
|
||||||
|
file { '/srv/static':
|
||||||
|
ensure => directory,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
apache::vhost { "logs.${domain}":
|
||||||
|
port => 80,
|
||||||
|
priority => '50',
|
||||||
|
docroot => '/srv/static/logs',
|
||||||
|
require => File['/srv/static/logs'],
|
||||||
|
template => 'openstackci/logs.vhost.erb',
|
||||||
|
}
|
||||||
|
|
||||||
|
apache::vhost { "logs-dev.${domain}":
|
||||||
|
port => 80,
|
||||||
|
priority => '51',
|
||||||
|
docroot => '/srv/static/logs',
|
||||||
|
require => File['/srv/static/logs'],
|
||||||
|
template => 'openstackci/logs-dev.vhost.erb',
|
||||||
|
}
|
||||||
|
|
||||||
|
file { '/srv/static/logs':
|
||||||
|
ensure => directory,
|
||||||
|
owner => 'jenkins',
|
||||||
|
group => 'jenkins',
|
||||||
|
require => User['jenkins'],
|
||||||
|
}
|
||||||
|
|
||||||
|
file { '/srv/static/logs/robots.txt':
|
||||||
|
ensure => present,
|
||||||
|
owner => 'root',
|
||||||
|
group => 'root',
|
||||||
|
mode => '0444',
|
||||||
|
source => 'puppet:///modules/openstackci/disallow_robots.txt',
|
||||||
|
require => File['/srv/static/logs'],
|
||||||
|
}
|
||||||
|
|
||||||
|
package { 'keyring':
|
||||||
|
ensure => 'latest',
|
||||||
|
provider => 'pip',
|
||||||
|
}
|
||||||
|
|
||||||
|
vcsrepo { '/opt/os-loganalyze':
|
||||||
|
ensure => latest,
|
||||||
|
provider => git,
|
||||||
|
revision => 'master',
|
||||||
|
source => 'https://git.openstack.org/openstack-infra/os-loganalyze',
|
||||||
|
require => Package['keyring'],
|
||||||
|
}
|
||||||
|
|
||||||
|
exec { 'install_os-loganalyze':
|
||||||
|
command => 'pip install .',
|
||||||
|
cwd => '/opt/os-loganalyze',
|
||||||
|
path => '/usr/local/bin:/usr/bin:/bin/',
|
||||||
|
refreshonly => true,
|
||||||
|
subscribe => Vcsrepo['/opt/os-loganalyze'],
|
||||||
|
}
|
||||||
|
|
||||||
|
file { '/etc/os_loganalyze':
|
||||||
|
ensure => directory,
|
||||||
|
owner => 'root',
|
||||||
|
group => 'root',
|
||||||
|
mode => '0755',
|
||||||
|
require => Vcsrepo['/opt/os-loganalyze'],
|
||||||
|
}
|
||||||
|
|
||||||
|
file { '/etc/os_loganalyze/wsgi.conf':
|
||||||
|
ensure => present,
|
||||||
|
owner => 'root',
|
||||||
|
group => 'www-data',
|
||||||
|
mode => '0440',
|
||||||
|
content => template('openstackci/os-loganalyze-wsgi.conf.erb'),
|
||||||
|
require => File['/etc/os_loganalyze'],
|
||||||
|
}
|
||||||
|
|
||||||
|
vcsrepo { '/opt/devstack-gate':
|
||||||
|
ensure => latest,
|
||||||
|
provider => git,
|
||||||
|
revision => 'master',
|
||||||
|
source => 'https://git.openstack.org/openstack-infra/devstack-gate',
|
||||||
|
}
|
||||||
|
|
||||||
|
file { '/srv/static/logs/help':
|
||||||
|
ensure => directory,
|
||||||
|
owner => 'root',
|
||||||
|
group => 'root',
|
||||||
|
mode => '0755',
|
||||||
|
require => File['/srv/static/logs'],
|
||||||
|
}
|
||||||
|
|
||||||
|
file { '/srv/static/logs/help/tempest-logs.html':
|
||||||
|
ensure => present,
|
||||||
|
owner => 'root',
|
||||||
|
group => 'root',
|
||||||
|
mode => '0444',
|
||||||
|
source => 'file:///opt/devstack-gate/help/tempest-logs.html',
|
||||||
|
require => [File['/srv/static/logs/help'], Vcsrepo['/opt/devstack-gate']],
|
||||||
|
}
|
||||||
|
|
||||||
|
file { '/srv/static/logs/help/tempest-overview.html':
|
||||||
|
ensure => present,
|
||||||
|
owner => 'root',
|
||||||
|
group => 'root',
|
||||||
|
mode => '0444',
|
||||||
|
source => 'file:///opt/devstack-gate/help/tempest-overview.html',
|
||||||
|
require => [File['/srv/static/logs/help'], Vcsrepo['/opt/devstack-gate']],
|
||||||
|
}
|
||||||
|
|
||||||
|
file { '/usr/local/sbin/log_archive_maintenance.sh':
|
||||||
|
ensure => present,
|
||||||
|
owner => 'root',
|
||||||
|
group => 'root',
|
||||||
|
mode => '0744',
|
||||||
|
source => 'puppet:///modules/openstackci/log_archive_maintenance.sh',
|
||||||
|
}
|
||||||
|
|
||||||
|
cron { 'gziprmlogs':
|
||||||
|
user => 'root',
|
||||||
|
minute => '0',
|
||||||
|
hour => '7',
|
||||||
|
weekday => '6',
|
||||||
|
command => 'bash /usr/local/sbin/log_archive_maintenance.sh',
|
||||||
|
environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin',
|
||||||
|
require => File['/usr/local/sbin/log_archive_maintenance.sh'],
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -8,5 +8,8 @@
|
||||||
"project_page": "http://ci.openstack.org/",
|
"project_page": "http://ci.openstack.org/",
|
||||||
"issues_url": "https://storyboard.openstack.org/#!/project/808",
|
"issues_url": "https://storyboard.openstack.org/#!/project/808",
|
||||||
"dependencies": [
|
"dependencies": [
|
||||||
|
{ "name": "puppetlabs/apache" },
|
||||||
|
{ "name": "openstackinfra/vcsrepo" },
|
||||||
|
{ "name": "openstackinfra/jenkins" }
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,88 @@
|
||||||
|
# -*- apache -*-
|
||||||
|
# ************************************
|
||||||
|
# Managed by Puppet
|
||||||
|
# ************************************
|
||||||
|
|
||||||
|
NameVirtualHost <%= @vhost_name %>:<%= @port %>
|
||||||
|
<VirtualHost <%= @vhost_name %>:<%= @port %>>
|
||||||
|
ServerName <%= @srvname %>
|
||||||
|
<% if @serveraliases.is_a? Array -%>
|
||||||
|
<% @serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%>
|
||||||
|
<% elsif @serveraliases != '' -%>
|
||||||
|
<%= " ServerAlias #{@serveraliases}" %>
|
||||||
|
<% end -%>
|
||||||
|
DocumentRoot <%= @docroot %>
|
||||||
|
|
||||||
|
# use Apache to compress the results afterwards, to save on the wire
|
||||||
|
# it's approx 18x savings of wire traffic to compress. We need to
|
||||||
|
# compress by content types that htmlify can produce
|
||||||
|
AddOutputFilterByType DEFLATE text/plain text/html
|
||||||
|
|
||||||
|
<FilesMatch \.html\.gz$>
|
||||||
|
ForceType text/html
|
||||||
|
AddDefaultCharset UTF-8
|
||||||
|
AddEncoding x-gzip gz
|
||||||
|
</FilesMatch>
|
||||||
|
<Directory <%= @docroot %>>
|
||||||
|
Options <%= @options %>
|
||||||
|
AllowOverride None
|
||||||
|
Order allow,deny
|
||||||
|
allow from all
|
||||||
|
Satisfy Any
|
||||||
|
</Directory>
|
||||||
|
<Directory /usr/local/lib/python2.7/dist-packages/os_loganalyze>
|
||||||
|
Allow from all
|
||||||
|
Satisfy Any
|
||||||
|
</Directory>
|
||||||
|
|
||||||
|
<Directory <%= @docroot %>/*/*/*/*/*-tempest-dsvm*/*>
|
||||||
|
ReadmeName /help/tempest-overview.html
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= @docroot %>/periodic*/*/*-tempest-dsvm*/*>
|
||||||
|
ReadmeName /help/tempest-overview.html
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= @docroot %>/*/*/*/*/*-tempest-dsvm*/*/logs/>
|
||||||
|
ReadmeName /help/tempest-logs.html
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= @docroot %>/periodic*/*/*-tempest-dsvm*/*/logs/>
|
||||||
|
ReadmeName /help/tempest-logs.html
|
||||||
|
</Directory>
|
||||||
|
|
||||||
|
# old patterns, can be removed in July 2014
|
||||||
|
<Directory <%= @docroot %>/*/*/*/*/gate-tempest-devstack*/*>
|
||||||
|
ReadmeName /help/tempest-overview.html
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= @docroot %>/periodic/*/periodic-tempest-devstack*/*>
|
||||||
|
ReadmeName /help/tempest-overview.html
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= @docroot %>/*/*/*/*/gate-tempest-devstack*/*/logs/>
|
||||||
|
ReadmeName /help/tempest-logs.html
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= @docroot %>/periodic/*/periodic-tempest-devstack*/*/logs/>
|
||||||
|
ReadmeName /help/tempest-logs.html
|
||||||
|
</Directory>
|
||||||
|
|
||||||
|
RewriteEngine On
|
||||||
|
# rewrite all txt.gz & html.gz files to map to our internal htmlify wsgi app
|
||||||
|
# PT, Pass-through: to come back around and get picked up by the WSGIScriptAlias
|
||||||
|
# NS, No-subrequest: on coming back through, mod-autoindex may have added index.html
|
||||||
|
# which would match the !-f condition. We therefore ensure the
|
||||||
|
# rewrite doesn't trigger by disallowing subrequests.
|
||||||
|
RewriteRule ^/(.*\.txt\.gz)$ /htmlify/$1 [QSA,L,PT,NS]
|
||||||
|
RewriteRule ^/(.*console\.html(\.gz)?)$ /htmlify/$1 [QSA,L,PT,NS]
|
||||||
|
|
||||||
|
# Check if the request exists as a file, directory or symbolic link
|
||||||
|
# If not, write the request to htmlify to see if we can fetch from swift
|
||||||
|
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-f
|
||||||
|
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-d
|
||||||
|
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-l
|
||||||
|
RewriteCond %{REQUEST_FILENAME} !^/icon
|
||||||
|
RewriteRule ^/(.*)$ /htmlify/$1 [QSA,L,PT,NS]
|
||||||
|
|
||||||
|
WSGIScriptAlias /htmlify /usr/local/lib/python2.7/dist-packages/os_loganalyze/wsgi.py
|
||||||
|
|
||||||
|
ErrorLog /var/log/apache2/<%= @name %>_error.log
|
||||||
|
LogLevel warn
|
||||||
|
CustomLog /var/log/apache2/<%= @name %>_access.log combined
|
||||||
|
ServerSignature Off
|
||||||
|
</VirtualHost>
|
|
@ -0,0 +1,91 @@
|
||||||
|
# -*- apache -*-
|
||||||
|
# ************************************
|
||||||
|
# Managed by Puppet
|
||||||
|
# ************************************
|
||||||
|
|
||||||
|
NameVirtualHost <%= vhost_name %>:<%= port %>
|
||||||
|
<VirtualHost <%= vhost_name %>:<%= port %>>
|
||||||
|
ServerName <%= srvname %>
|
||||||
|
<% if serveraliases.is_a? Array -%>
|
||||||
|
<% serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%>
|
||||||
|
<% elsif serveraliases != '' -%>
|
||||||
|
<%= " ServerAlias #{serveraliases}" %>
|
||||||
|
<% end -%>
|
||||||
|
DocumentRoot <%= docroot %>
|
||||||
|
|
||||||
|
# use Apache to compress the results afterwards, to save on the wire
|
||||||
|
# it's approx 18x savings of wire traffic to compress. We need to
|
||||||
|
# compress by content types that htmlify can produce
|
||||||
|
AddOutputFilterByType DEFLATE text/plain text/html
|
||||||
|
|
||||||
|
<FilesMatch \.html\.gz$>
|
||||||
|
ForceType text/html
|
||||||
|
AddDefaultCharset UTF-8
|
||||||
|
AddEncoding x-gzip gz
|
||||||
|
</FilesMatch>
|
||||||
|
<Directory <%= docroot %>>
|
||||||
|
Options <%= options %>
|
||||||
|
AllowOverride None
|
||||||
|
Order allow,deny
|
||||||
|
allow from all
|
||||||
|
Satisfy Any
|
||||||
|
</Directory>
|
||||||
|
<Directory /usr/local/lib/python2.7/dist-packages/os_loganalyze>
|
||||||
|
Allow from all
|
||||||
|
Satisfy Any
|
||||||
|
</Directory>
|
||||||
|
|
||||||
|
<Directory <%= docroot %>/*/*/*/*/*-tempest-dsvm*/*>
|
||||||
|
ReadmeName /help/tempest-overview.html
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= docroot %>/periodic*/*/*-tempest-dsvm*/*>
|
||||||
|
ReadmeName /help/tempest-overview.html
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= docroot %>/*/*/*/*/*-tempest-dsvm*/*/logs/>
|
||||||
|
ReadmeName /help/tempest-logs.html
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= docroot %>/periodic*/*/*-tempest-dsvm*/*/logs/>
|
||||||
|
ReadmeName /help/tempest-logs.html
|
||||||
|
</Directory>
|
||||||
|
|
||||||
|
# old patterns, can be removed in July 2014
|
||||||
|
<Directory <%= docroot %>/*/*/*/*/gate-tempest-devstack*/*>
|
||||||
|
ReadmeName /help/tempest-overview.html
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= docroot %>/periodic/*/periodic-tempest-devstack*/*>
|
||||||
|
ReadmeName /help/tempest-overview.html
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= docroot %>/*/*/*/*/gate-tempest-devstack*/*/logs/>
|
||||||
|
ReadmeName /help/tempest-logs.html
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= docroot %>/periodic/*/periodic-tempest-devstack*/*/logs/>
|
||||||
|
ReadmeName /help/tempest-logs.html
|
||||||
|
</Directory>
|
||||||
|
|
||||||
|
RewriteEngine On
|
||||||
|
# rewrite txt.gz & console.html[.gz] files to map to our internal htmlify
|
||||||
|
# wsgi app
|
||||||
|
# PT, Pass-through: to come back around and get picked up by the
|
||||||
|
# WSGIScriptAlias
|
||||||
|
# NS, No-subrequest: on coming back through, mod-autoindex may have added
|
||||||
|
# index.html which would match the !-f condition. We
|
||||||
|
# therefore ensure the rewrite doesn't trigger by
|
||||||
|
# disallowing subrequests.
|
||||||
|
RewriteRule ^/(.*\.txt\.gz)$ /htmlify/$1 [QSA,L,PT,NS]
|
||||||
|
RewriteRule ^/(.*console\.html(\.gz)?)$ /htmlify/$1 [QSA,L,PT,NS]
|
||||||
|
|
||||||
|
# Check if the request exists as a file, directory or symbolic link
|
||||||
|
# If not, write the request to htmlify to see if we can fetch from swift
|
||||||
|
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-f
|
||||||
|
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-d
|
||||||
|
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-l
|
||||||
|
RewriteCond %{REQUEST_FILENAME} !^/icon
|
||||||
|
RewriteRule ^/(.*)$ /htmlify/$1 [QSA,L,PT,NS]
|
||||||
|
|
||||||
|
WSGIScriptAlias /htmlify /usr/local/lib/python2.7/dist-packages/os_loganalyze/wsgi.py
|
||||||
|
|
||||||
|
ErrorLog /var/log/apache2/<%= name %>_error.log
|
||||||
|
LogLevel warn
|
||||||
|
CustomLog /var/log/apache2/<%= name %>_access.log combined
|
||||||
|
ServerSignature Off
|
||||||
|
</VirtualHost>
|
|
@ -0,0 +1,10 @@
|
||||||
|
<% if swift_authurl != "" -%>
|
||||||
|
[swift]
|
||||||
|
authurl=<%= swift_authurl %>
|
||||||
|
user=<%= swift_user %>
|
||||||
|
password=<%= swift_key %>
|
||||||
|
tenant=<%= swift_tenant_name %>
|
||||||
|
region=<%= swift_region_name %>
|
||||||
|
chunk_size=64
|
||||||
|
container=<%= swift_default_container %>
|
||||||
|
<% end -%>
|
Loading…
Reference in New Issue