Create a log server like http://logs.openstack.org
Copy the log server templates & files used in: https://git.openstack.org/cgit/openstack-infra/system-config/tree/modules/openstack_project Refactor the log server portion out of: https://git.openstack.org/cgit/openstack-infra/system-config/tree/modules/openstack_project/manifests/static.pp Additional modifications: 1. Add protections for portions that are likely to conflict. 2. Update os_loganalyzer install to use pip instead of python Related system-config change: I390f6ea8aaace276d211d55a0e17f25dd6ae26b5 Change-Id: Ib8476df83b7c5491158fa3cab588213be60fa2ab
This commit is contained in:
Ramy Asselin
parent
f822432cb1
commit
e8106a86e6
|
|
@ -0,0 +1,2 @@
|
|||
User-agent: *
|
||||
Disallow: /
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/sh
|
||||
sleep $((RANDOM%600)) && \
|
||||
flock -n /var/run/gziplogs.lock \
|
||||
find -O3 /srv/static/logs/ -depth -not -name robots.txt -not -name lost+found \
|
||||
-not -wholename /srv/static/logs/help/\* \( \
|
||||
\( -type f -mmin +10 -not -name \*\[.-\]gz -not -name \*\[._-\]\[zZ\] \
|
||||
\( -name \*.txt -or -name \*.html -or -name tmp\* \) \
|
||||
-exec gzip \{\} \; \) \
|
||||
-o \( -type f -mtime +120 -execdir rm \{\} \; \) \
|
||||
-o \( -type d -empty -mtime +1 -execdir rmdir {} \; \) \)
|
||||
find -O3 /srv/static/docs-draft/ -depth -not -name lost+found \( \
|
||||
\( -type f -mtime +21 -execdir rm \{\} \; \) \
|
||||
-o \( -type d -empty -mtime +1 -execdir rmdir {} \; \) \)
|
||||
|
|
@ -0,0 +1,183 @@
|
|||
# Copyright (c) 2012-2015 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# == Class: openstackci::logserver
|
||||
#
|
||||
class openstackci::logserver (
|
||||
$domain,
|
||||
$jenkins_ssh_key,
|
||||
$swift_authurl = '',
|
||||
$swift_user = '',
|
||||
$swift_key = '',
|
||||
$swift_tenant_name = '',
|
||||
$swift_region_name = '',
|
||||
$swift_default_container = '',
|
||||
) {
|
||||
|
||||
if ! defined(Class['::jenkins::jenkinsuser']) {
|
||||
class { '::jenkins::jenkinsuser':
|
||||
ssh_key => $jenkins_ssh_key,
|
||||
}
|
||||
}
|
||||
|
||||
include apache
|
||||
include apache::mod::wsgi
|
||||
|
||||
if ! defined(A2mod['rewrite']) {
|
||||
a2mod { 'rewrite':
|
||||
ensure => present,
|
||||
}
|
||||
}
|
||||
|
||||
if ! defined(A2mod['proxy']) {
|
||||
a2mod { 'proxy':
|
||||
ensure => present,
|
||||
}
|
||||
}
|
||||
|
||||
if ! defined(A2mod['proxy_http']) {
|
||||
a2mod { 'proxy_http':
|
||||
ensure => present,
|
||||
}
|
||||
}
|
||||
|
||||
if ! defined(File['/srv/static']) {
|
||||
file { '/srv/static':
|
||||
ensure => directory,
|
||||
}
|
||||
}
|
||||
|
||||
apache::vhost { "logs.${domain}":
|
||||
port => 80,
|
||||
priority => '50',
|
||||
docroot => '/srv/static/logs',
|
||||
require => File['/srv/static/logs'],
|
||||
template => 'openstackci/logs.vhost.erb',
|
||||
}
|
||||
|
||||
apache::vhost { "logs-dev.${domain}":
|
||||
port => 80,
|
||||
priority => '51',
|
||||
docroot => '/srv/static/logs',
|
||||
require => File['/srv/static/logs'],
|
||||
template => 'openstackci/logs-dev.vhost.erb',
|
||||
}
|
||||
|
||||
file { '/srv/static/logs':
|
||||
ensure => directory,
|
||||
owner => 'jenkins',
|
||||
group => 'jenkins',
|
||||
require => User['jenkins'],
|
||||
}
|
||||
|
||||
file { '/srv/static/logs/robots.txt':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0444',
|
||||
source => 'puppet:///modules/openstackci/disallow_robots.txt',
|
||||
require => File['/srv/static/logs'],
|
||||
}
|
||||
|
||||
package { 'keyring':
|
||||
ensure => 'latest',
|
||||
provider => 'pip',
|
||||
}
|
||||
|
||||
vcsrepo { '/opt/os-loganalyze':
|
||||
ensure => latest,
|
||||
provider => git,
|
||||
revision => 'master',
|
||||
source => 'https://git.openstack.org/openstack-infra/os-loganalyze',
|
||||
require => Package['keyring'],
|
||||
}
|
||||
|
||||
exec { 'install_os-loganalyze':
|
||||
command => 'pip install .',
|
||||
cwd => '/opt/os-loganalyze',
|
||||
path => '/usr/local/bin:/usr/bin:/bin/',
|
||||
refreshonly => true,
|
||||
subscribe => Vcsrepo['/opt/os-loganalyze'],
|
||||
}
|
||||
|
||||
file { '/etc/os_loganalyze':
|
||||
ensure => directory,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
require => Vcsrepo['/opt/os-loganalyze'],
|
||||
}
|
||||
|
||||
file { '/etc/os_loganalyze/wsgi.conf':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'www-data',
|
||||
mode => '0440',
|
||||
content => template('openstackci/os-loganalyze-wsgi.conf.erb'),
|
||||
require => File['/etc/os_loganalyze'],
|
||||
}
|
||||
|
||||
vcsrepo { '/opt/devstack-gate':
|
||||
ensure => latest,
|
||||
provider => git,
|
||||
revision => 'master',
|
||||
source => 'https://git.openstack.org/openstack-infra/devstack-gate',
|
||||
}
|
||||
|
||||
file { '/srv/static/logs/help':
|
||||
ensure => directory,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
require => File['/srv/static/logs'],
|
||||
}
|
||||
|
||||
file { '/srv/static/logs/help/tempest-logs.html':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0444',
|
||||
source => 'file:///opt/devstack-gate/help/tempest-logs.html',
|
||||
require => [File['/srv/static/logs/help'], Vcsrepo['/opt/devstack-gate']],
|
||||
}
|
||||
|
||||
file { '/srv/static/logs/help/tempest-overview.html':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0444',
|
||||
source => 'file:///opt/devstack-gate/help/tempest-overview.html',
|
||||
require => [File['/srv/static/logs/help'], Vcsrepo['/opt/devstack-gate']],
|
||||
}
|
||||
|
||||
file { '/usr/local/sbin/log_archive_maintenance.sh':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0744',
|
||||
source => 'puppet:///modules/openstackci/log_archive_maintenance.sh',
|
||||
}
|
||||
|
||||
cron { 'gziprmlogs':
|
||||
user => 'root',
|
||||
minute => '0',
|
||||
hour => '7',
|
||||
weekday => '6',
|
||||
command => 'bash /usr/local/sbin/log_archive_maintenance.sh',
|
||||
environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin',
|
||||
require => File['/usr/local/sbin/log_archive_maintenance.sh'],
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -8,5 +8,8 @@
|
|||
"project_page": "http://ci.openstack.org/",
|
||||
"issues_url": "https://storyboard.openstack.org/#!/project/808",
|
||||
"dependencies": [
|
||||
{ "name": "puppetlabs/apache" },
|
||||
{ "name": "openstackinfra/vcsrepo" },
|
||||
{ "name": "openstackinfra/jenkins" }
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,88 @@
|
|||
# -*- apache -*-
|
||||
# ************************************
|
||||
# Managed by Puppet
|
||||
# ************************************
|
||||
|
||||
NameVirtualHost <%= @vhost_name %>:<%= @port %>
|
||||
<VirtualHost <%= @vhost_name %>:<%= @port %>>
|
||||
ServerName <%= @srvname %>
|
||||
<% if @serveraliases.is_a? Array -%>
|
||||
<% @serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%>
|
||||
<% elsif @serveraliases != '' -%>
|
||||
<%= " ServerAlias #{@serveraliases}" %>
|
||||
<% end -%>
|
||||
DocumentRoot <%= @docroot %>
|
||||
|
||||
# use Apache to compress the results afterwards, to save on the wire
|
||||
# it's approx 18x savings of wire traffic to compress. We need to
|
||||
# compress by content types that htmlify can produce
|
||||
AddOutputFilterByType DEFLATE text/plain text/html
|
||||
|
||||
<FilesMatch \.html\.gz$>
|
||||
ForceType text/html
|
||||
AddDefaultCharset UTF-8
|
||||
AddEncoding x-gzip gz
|
||||
</FilesMatch>
|
||||
<Directory <%= @docroot %>>
|
||||
Options <%= @options %>
|
||||
AllowOverride None
|
||||
Order allow,deny
|
||||
allow from all
|
||||
Satisfy Any
|
||||
</Directory>
|
||||
<Directory /usr/local/lib/python2.7/dist-packages/os_loganalyze>
|
||||
Allow from all
|
||||
Satisfy Any
|
||||
</Directory>
|
||||
|
||||
<Directory <%= @docroot %>/*/*/*/*/*-tempest-dsvm*/*>
|
||||
ReadmeName /help/tempest-overview.html
|
||||
</Directory>
|
||||
<Directory <%= @docroot %>/periodic*/*/*-tempest-dsvm*/*>
|
||||
ReadmeName /help/tempest-overview.html
|
||||
</Directory>
|
||||
<Directory <%= @docroot %>/*/*/*/*/*-tempest-dsvm*/*/logs/>
|
||||
ReadmeName /help/tempest-logs.html
|
||||
</Directory>
|
||||
<Directory <%= @docroot %>/periodic*/*/*-tempest-dsvm*/*/logs/>
|
||||
ReadmeName /help/tempest-logs.html
|
||||
</Directory>
|
||||
|
||||
# old patterns, can be removed in July 2014
|
||||
<Directory <%= @docroot %>/*/*/*/*/gate-tempest-devstack*/*>
|
||||
ReadmeName /help/tempest-overview.html
|
||||
</Directory>
|
||||
<Directory <%= @docroot %>/periodic/*/periodic-tempest-devstack*/*>
|
||||
ReadmeName /help/tempest-overview.html
|
||||
</Directory>
|
||||
<Directory <%= @docroot %>/*/*/*/*/gate-tempest-devstack*/*/logs/>
|
||||
ReadmeName /help/tempest-logs.html
|
||||
</Directory>
|
||||
<Directory <%= @docroot %>/periodic/*/periodic-tempest-devstack*/*/logs/>
|
||||
ReadmeName /help/tempest-logs.html
|
||||
</Directory>
|
||||
|
||||
RewriteEngine On
|
||||
# rewrite all txt.gz & html.gz files to map to our internal htmlify wsgi app
|
||||
# PT, Pass-through: to come back around and get picked up by the WSGIScriptAlias
|
||||
# NS, No-subrequest: on coming back through, mod-autoindex may have added index.html
|
||||
# which would match the !-f condition. We therefore ensure the
|
||||
# rewrite doesn't trigger by disallowing subrequests.
|
||||
RewriteRule ^/(.*\.txt\.gz)$ /htmlify/$1 [QSA,L,PT,NS]
|
||||
RewriteRule ^/(.*console\.html(\.gz)?)$ /htmlify/$1 [QSA,L,PT,NS]
|
||||
|
||||
# Check if the request exists as a file, directory or symbolic link
|
||||
# If not, write the request to htmlify to see if we can fetch from swift
|
||||
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-f
|
||||
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-d
|
||||
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-l
|
||||
RewriteCond %{REQUEST_FILENAME} !^/icon
|
||||
RewriteRule ^/(.*)$ /htmlify/$1 [QSA,L,PT,NS]
|
||||
|
||||
WSGIScriptAlias /htmlify /usr/local/lib/python2.7/dist-packages/os_loganalyze/wsgi.py
|
||||
|
||||
ErrorLog /var/log/apache2/<%= @name %>_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/apache2/<%= @name %>_access.log combined
|
||||
ServerSignature Off
|
||||
</VirtualHost>
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
# -*- apache -*-
|
||||
# ************************************
|
||||
# Managed by Puppet
|
||||
# ************************************
|
||||
|
||||
NameVirtualHost <%= vhost_name %>:<%= port %>
|
||||
<VirtualHost <%= vhost_name %>:<%= port %>>
|
||||
ServerName <%= srvname %>
|
||||
<% if serveraliases.is_a? Array -%>
|
||||
<% serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%>
|
||||
<% elsif serveraliases != '' -%>
|
||||
<%= " ServerAlias #{serveraliases}" %>
|
||||
<% end -%>
|
||||
DocumentRoot <%= docroot %>
|
||||
|
||||
# use Apache to compress the results afterwards, to save on the wire
|
||||
# it's approx 18x savings of wire traffic to compress. We need to
|
||||
# compress by content types that htmlify can produce
|
||||
AddOutputFilterByType DEFLATE text/plain text/html
|
||||
|
||||
<FilesMatch \.html\.gz$>
|
||||
ForceType text/html
|
||||
AddDefaultCharset UTF-8
|
||||
AddEncoding x-gzip gz
|
||||
</FilesMatch>
|
||||
<Directory <%= docroot %>>
|
||||
Options <%= options %>
|
||||
AllowOverride None
|
||||
Order allow,deny
|
||||
allow from all
|
||||
Satisfy Any
|
||||
</Directory>
|
||||
<Directory /usr/local/lib/python2.7/dist-packages/os_loganalyze>
|
||||
Allow from all
|
||||
Satisfy Any
|
||||
</Directory>
|
||||
|
||||
<Directory <%= docroot %>/*/*/*/*/*-tempest-dsvm*/*>
|
||||
ReadmeName /help/tempest-overview.html
|
||||
</Directory>
|
||||
<Directory <%= docroot %>/periodic*/*/*-tempest-dsvm*/*>
|
||||
ReadmeName /help/tempest-overview.html
|
||||
</Directory>
|
||||
<Directory <%= docroot %>/*/*/*/*/*-tempest-dsvm*/*/logs/>
|
||||
ReadmeName /help/tempest-logs.html
|
||||
</Directory>
|
||||
<Directory <%= docroot %>/periodic*/*/*-tempest-dsvm*/*/logs/>
|
||||
ReadmeName /help/tempest-logs.html
|
||||
</Directory>
|
||||
|
||||
# old patterns, can be removed in July 2014
|
||||
<Directory <%= docroot %>/*/*/*/*/gate-tempest-devstack*/*>
|
||||
ReadmeName /help/tempest-overview.html
|
||||
</Directory>
|
||||
<Directory <%= docroot %>/periodic/*/periodic-tempest-devstack*/*>
|
||||
ReadmeName /help/tempest-overview.html
|
||||
</Directory>
|
||||
<Directory <%= docroot %>/*/*/*/*/gate-tempest-devstack*/*/logs/>
|
||||
ReadmeName /help/tempest-logs.html
|
||||
</Directory>
|
||||
<Directory <%= docroot %>/periodic/*/periodic-tempest-devstack*/*/logs/>
|
||||
ReadmeName /help/tempest-logs.html
|
||||
</Directory>
|
||||
|
||||
RewriteEngine On
|
||||
# rewrite txt.gz & console.html[.gz] files to map to our internal htmlify
|
||||
# wsgi app
|
||||
# PT, Pass-through: to come back around and get picked up by the
|
||||
# WSGIScriptAlias
|
||||
# NS, No-subrequest: on coming back through, mod-autoindex may have added
|
||||
# index.html which would match the !-f condition. We
|
||||
# therefore ensure the rewrite doesn't trigger by
|
||||
# disallowing subrequests.
|
||||
RewriteRule ^/(.*\.txt\.gz)$ /htmlify/$1 [QSA,L,PT,NS]
|
||||
RewriteRule ^/(.*console\.html(\.gz)?)$ /htmlify/$1 [QSA,L,PT,NS]
|
||||
|
||||
# Check if the request exists as a file, directory or symbolic link
|
||||
# If not, write the request to htmlify to see if we can fetch from swift
|
||||
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-f
|
||||
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-d
|
||||
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-l
|
||||
RewriteCond %{REQUEST_FILENAME} !^/icon
|
||||
RewriteRule ^/(.*)$ /htmlify/$1 [QSA,L,PT,NS]
|
||||
|
||||
WSGIScriptAlias /htmlify /usr/local/lib/python2.7/dist-packages/os_loganalyze/wsgi.py
|
||||
|
||||
ErrorLog /var/log/apache2/<%= name %>_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/apache2/<%= name %>_access.log combined
|
||||
ServerSignature Off
|
||||
</VirtualHost>
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
<% if swift_authurl != "" -%>
|
||||
[swift]
|
||||
authurl=<%= swift_authurl %>
|
||||
user=<%= swift_user %>
|
||||
password=<%= swift_key %>
|
||||
tenant=<%= swift_tenant_name %>
|
||||
region=<%= swift_region_name %>
|
||||
chunk_size=64
|
||||
container=<%= swift_default_container %>
|
||||
<% end -%>
|
||||
Loading…
Reference in New Issue