From e8106a86e6f2507a44e2ccc7d2f137bd6b413a6f Mon Sep 17 00:00:00 2001 From: Ramy Asselin Date: Tue, 24 Mar 2015 15:50:51 -0700 Subject: [PATCH] Create a log server like http://logs.openstack.org Copy the log server templates & files used in: https://git.openstack.org/cgit/openstack-infra/system-config/tree/modules/openstack_project Refactor the log server portion out of: https://git.openstack.org/cgit/openstack-infra/system-config/tree/modules/openstack_project/manifests/static.pp Additional modifications: 1. Add protections for portions that are likely to conflict. 2. Update os_loganalyzer install to use pip instead of python Related system-config change: I390f6ea8aaace276d211d55a0e17f25dd6ae26b5 Change-Id: Ib8476df83b7c5491158fa3cab588213be60fa2ab --- files/disallow_robots.txt | 2 + files/log_archive_maintenance.sh | 13 ++ manifests/logserver.pp | 183 ++++++++++++++++++++++++++ metadata.json | 3 + templates/logs-dev.vhost.erb | 88 +++++++++++++ templates/logs.vhost.erb | 91 +++++++++++++ templates/os-loganalyze-wsgi.conf.erb | 10 ++ 7 files changed, 390 insertions(+) create mode 100644 files/disallow_robots.txt create mode 100644 files/log_archive_maintenance.sh create mode 100644 manifests/logserver.pp create mode 100644 templates/logs-dev.vhost.erb create mode 100644 templates/logs.vhost.erb create mode 100644 templates/os-loganalyze-wsgi.conf.erb diff --git a/files/disallow_robots.txt b/files/disallow_robots.txt new file mode 100644 index 0000000..1f53798 --- /dev/null +++ b/files/disallow_robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/files/log_archive_maintenance.sh b/files/log_archive_maintenance.sh new file mode 100644 index 0000000..71af22a --- /dev/null +++ b/files/log_archive_maintenance.sh @@ -0,0 +1,13 @@ +#!/bin/sh +sleep $((RANDOM%600)) && \ +flock -n /var/run/gziplogs.lock \ +find -O3 /srv/static/logs/ -depth -not -name robots.txt -not -name lost+found \ + -not -wholename /srv/static/logs/help/\* \( \ + \( -type f -mmin +10 -not -name \*\[.-\]gz -not -name \*\[._-\]\[zZ\] \ + \( -name \*.txt -or -name \*.html -or -name tmp\* \) \ + -exec gzip \{\} \; \) \ + -o \( -type f -mtime +120 -execdir rm \{\} \; \) \ + -o \( -type d -empty -mtime +1 -execdir rmdir {} \; \) \) +find -O3 /srv/static/docs-draft/ -depth -not -name lost+found \( \ + \( -type f -mtime +21 -execdir rm \{\} \; \) \ + -o \( -type d -empty -mtime +1 -execdir rmdir {} \; \) \) diff --git a/manifests/logserver.pp b/manifests/logserver.pp new file mode 100644 index 0000000..245fe2d --- /dev/null +++ b/manifests/logserver.pp @@ -0,0 +1,183 @@ +# Copyright (c) 2012-2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# == Class: openstackci::logserver +# +class openstackci::logserver ( + $domain, + $jenkins_ssh_key, + $swift_authurl = '', + $swift_user = '', + $swift_key = '', + $swift_tenant_name = '', + $swift_region_name = '', + $swift_default_container = '', +) { + + if ! defined(Class['::jenkins::jenkinsuser']) { + class { '::jenkins::jenkinsuser': + ssh_key => $jenkins_ssh_key, + } + } + + include apache + include apache::mod::wsgi + + if ! defined(A2mod['rewrite']) { + a2mod { 'rewrite': + ensure => present, + } + } + + if ! defined(A2mod['proxy']) { + a2mod { 'proxy': + ensure => present, + } + } + + if ! defined(A2mod['proxy_http']) { + a2mod { 'proxy_http': + ensure => present, + } + } + + if ! defined(File['/srv/static']) { + file { '/srv/static': + ensure => directory, + } + } + + apache::vhost { "logs.${domain}": + port => 80, + priority => '50', + docroot => '/srv/static/logs', + require => File['/srv/static/logs'], + template => 'openstackci/logs.vhost.erb', + } + + apache::vhost { "logs-dev.${domain}": + port => 80, + priority => '51', + docroot => '/srv/static/logs', + require => File['/srv/static/logs'], + template => 'openstackci/logs-dev.vhost.erb', + } + + file { '/srv/static/logs': + ensure => directory, + owner => 'jenkins', + group => 'jenkins', + require => User['jenkins'], + } + + file { '/srv/static/logs/robots.txt': + ensure => present, + owner => 'root', + group => 'root', + mode => '0444', + source => 'puppet:///modules/openstackci/disallow_robots.txt', + require => File['/srv/static/logs'], + } + + package { 'keyring': + ensure => 'latest', + provider => 'pip', + } + + vcsrepo { '/opt/os-loganalyze': + ensure => latest, + provider => git, + revision => 'master', + source => 'https://git.openstack.org/openstack-infra/os-loganalyze', + require => Package['keyring'], + } + + exec { 'install_os-loganalyze': + command => 'pip install .', + cwd => '/opt/os-loganalyze', + path => '/usr/local/bin:/usr/bin:/bin/', + refreshonly => true, + subscribe => Vcsrepo['/opt/os-loganalyze'], + } + + file { '/etc/os_loganalyze': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0755', + require => Vcsrepo['/opt/os-loganalyze'], + } + + file { '/etc/os_loganalyze/wsgi.conf': + ensure => present, + owner => 'root', + group => 'www-data', + mode => '0440', + content => template('openstackci/os-loganalyze-wsgi.conf.erb'), + require => File['/etc/os_loganalyze'], + } + + vcsrepo { '/opt/devstack-gate': + ensure => latest, + provider => git, + revision => 'master', + source => 'https://git.openstack.org/openstack-infra/devstack-gate', + } + + file { '/srv/static/logs/help': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0755', + require => File['/srv/static/logs'], + } + + file { '/srv/static/logs/help/tempest-logs.html': + ensure => present, + owner => 'root', + group => 'root', + mode => '0444', + source => 'file:///opt/devstack-gate/help/tempest-logs.html', + require => [File['/srv/static/logs/help'], Vcsrepo['/opt/devstack-gate']], + } + + file { '/srv/static/logs/help/tempest-overview.html': + ensure => present, + owner => 'root', + group => 'root', + mode => '0444', + source => 'file:///opt/devstack-gate/help/tempest-overview.html', + require => [File['/srv/static/logs/help'], Vcsrepo['/opt/devstack-gate']], + } + + file { '/usr/local/sbin/log_archive_maintenance.sh': + ensure => present, + owner => 'root', + group => 'root', + mode => '0744', + source => 'puppet:///modules/openstackci/log_archive_maintenance.sh', + } + + cron { 'gziprmlogs': + user => 'root', + minute => '0', + hour => '7', + weekday => '6', + command => 'bash /usr/local/sbin/log_archive_maintenance.sh', + environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin', + require => File['/usr/local/sbin/log_archive_maintenance.sh'], + } + +} diff --git a/metadata.json b/metadata.json index b376fda..8d414e1 100644 --- a/metadata.json +++ b/metadata.json @@ -8,5 +8,8 @@ "project_page": "http://ci.openstack.org/", "issues_url": "https://storyboard.openstack.org/#!/project/808", "dependencies": [ + { "name": "puppetlabs/apache" }, + { "name": "openstackinfra/vcsrepo" }, + { "name": "openstackinfra/jenkins" } ] } diff --git a/templates/logs-dev.vhost.erb b/templates/logs-dev.vhost.erb new file mode 100644 index 0000000..48d6e1b --- /dev/null +++ b/templates/logs-dev.vhost.erb @@ -0,0 +1,88 @@ +# -*- apache -*- +# ************************************ +# Managed by Puppet +# ************************************ + +NameVirtualHost <%= @vhost_name %>:<%= @port %> +:<%= @port %>> + ServerName <%= @srvname %> +<% if @serveraliases.is_a? Array -%> +<% @serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%> +<% elsif @serveraliases != '' -%> +<%= " ServerAlias #{@serveraliases}" %> +<% end -%> + DocumentRoot <%= @docroot %> + + # use Apache to compress the results afterwards, to save on the wire + # it's approx 18x savings of wire traffic to compress. We need to + # compress by content types that htmlify can produce + AddOutputFilterByType DEFLATE text/plain text/html + + + ForceType text/html + AddDefaultCharset UTF-8 + AddEncoding x-gzip gz + + > + Options <%= @options %> + AllowOverride None + Order allow,deny + allow from all + Satisfy Any + + + Allow from all + Satisfy Any + + + /*/*/*/*/*-tempest-dsvm*/*> + ReadmeName /help/tempest-overview.html + + /periodic*/*/*-tempest-dsvm*/*> + ReadmeName /help/tempest-overview.html + + /*/*/*/*/*-tempest-dsvm*/*/logs/> + ReadmeName /help/tempest-logs.html + + /periodic*/*/*-tempest-dsvm*/*/logs/> + ReadmeName /help/tempest-logs.html + + + # old patterns, can be removed in July 2014 + /*/*/*/*/gate-tempest-devstack*/*> + ReadmeName /help/tempest-overview.html + + /periodic/*/periodic-tempest-devstack*/*> + ReadmeName /help/tempest-overview.html + + /*/*/*/*/gate-tempest-devstack*/*/logs/> + ReadmeName /help/tempest-logs.html + + /periodic/*/periodic-tempest-devstack*/*/logs/> + ReadmeName /help/tempest-logs.html + + + RewriteEngine On + # rewrite all txt.gz & html.gz files to map to our internal htmlify wsgi app + # PT, Pass-through: to come back around and get picked up by the WSGIScriptAlias + # NS, No-subrequest: on coming back through, mod-autoindex may have added index.html + # which would match the !-f condition. We therefore ensure the + # rewrite doesn't trigger by disallowing subrequests. + RewriteRule ^/(.*\.txt\.gz)$ /htmlify/$1 [QSA,L,PT,NS] + RewriteRule ^/(.*console\.html(\.gz)?)$ /htmlify/$1 [QSA,L,PT,NS] + + # Check if the request exists as a file, directory or symbolic link + # If not, write the request to htmlify to see if we can fetch from swift + RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-f + RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-d + RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-l + RewriteCond %{REQUEST_FILENAME} !^/icon + RewriteRule ^/(.*)$ /htmlify/$1 [QSA,L,PT,NS] + + WSGIScriptAlias /htmlify /usr/local/lib/python2.7/dist-packages/os_loganalyze/wsgi.py + + ErrorLog /var/log/apache2/<%= @name %>_error.log + LogLevel warn + CustomLog /var/log/apache2/<%= @name %>_access.log combined + ServerSignature Off + diff --git a/templates/logs.vhost.erb b/templates/logs.vhost.erb new file mode 100644 index 0000000..1bec5be --- /dev/null +++ b/templates/logs.vhost.erb @@ -0,0 +1,91 @@ +# -*- apache -*- +# ************************************ +# Managed by Puppet +# ************************************ + +NameVirtualHost <%= vhost_name %>:<%= port %> +:<%= port %>> + ServerName <%= srvname %> +<% if serveraliases.is_a? Array -%> +<% serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%> +<% elsif serveraliases != '' -%> +<%= " ServerAlias #{serveraliases}" %> +<% end -%> + DocumentRoot <%= docroot %> + + # use Apache to compress the results afterwards, to save on the wire + # it's approx 18x savings of wire traffic to compress. We need to + # compress by content types that htmlify can produce + AddOutputFilterByType DEFLATE text/plain text/html + + + ForceType text/html + AddDefaultCharset UTF-8 + AddEncoding x-gzip gz + + > + Options <%= options %> + AllowOverride None + Order allow,deny + allow from all + Satisfy Any + + + Allow from all + Satisfy Any + + + /*/*/*/*/*-tempest-dsvm*/*> + ReadmeName /help/tempest-overview.html + + /periodic*/*/*-tempest-dsvm*/*> + ReadmeName /help/tempest-overview.html + + /*/*/*/*/*-tempest-dsvm*/*/logs/> + ReadmeName /help/tempest-logs.html + + /periodic*/*/*-tempest-dsvm*/*/logs/> + ReadmeName /help/tempest-logs.html + + + # old patterns, can be removed in July 2014 + /*/*/*/*/gate-tempest-devstack*/*> + ReadmeName /help/tempest-overview.html + + /periodic/*/periodic-tempest-devstack*/*> + ReadmeName /help/tempest-overview.html + + /*/*/*/*/gate-tempest-devstack*/*/logs/> + ReadmeName /help/tempest-logs.html + + /periodic/*/periodic-tempest-devstack*/*/logs/> + ReadmeName /help/tempest-logs.html + + + RewriteEngine On + # rewrite txt.gz & console.html[.gz] files to map to our internal htmlify + # wsgi app + # PT, Pass-through: to come back around and get picked up by the + # WSGIScriptAlias + # NS, No-subrequest: on coming back through, mod-autoindex may have added + # index.html which would match the !-f condition. We + # therefore ensure the rewrite doesn't trigger by + # disallowing subrequests. + RewriteRule ^/(.*\.txt\.gz)$ /htmlify/$1 [QSA,L,PT,NS] + RewriteRule ^/(.*console\.html(\.gz)?)$ /htmlify/$1 [QSA,L,PT,NS] + + # Check if the request exists as a file, directory or symbolic link + # If not, write the request to htmlify to see if we can fetch from swift + RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-f + RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-d + RewriteCond %{DOCUMENT_ROOT}%{REQUEST_FILENAME} !-l + RewriteCond %{REQUEST_FILENAME} !^/icon + RewriteRule ^/(.*)$ /htmlify/$1 [QSA,L,PT,NS] + + WSGIScriptAlias /htmlify /usr/local/lib/python2.7/dist-packages/os_loganalyze/wsgi.py + + ErrorLog /var/log/apache2/<%= name %>_error.log + LogLevel warn + CustomLog /var/log/apache2/<%= name %>_access.log combined + ServerSignature Off + diff --git a/templates/os-loganalyze-wsgi.conf.erb b/templates/os-loganalyze-wsgi.conf.erb new file mode 100644 index 0000000..a917670 --- /dev/null +++ b/templates/os-loganalyze-wsgi.conf.erb @@ -0,0 +1,10 @@ +<% if swift_authurl != "" -%> +[swift] +authurl=<%= swift_authurl %> +user=<%= swift_user %> +password=<%= swift_key %> +tenant=<%= swift_tenant_name %> +region=<%= swift_region_name %> +chunk_size=64 +container=<%= swift_default_container %> +<% end -%>