openstack/fuel-library
Code Issues Proposed changes

Rework logrotate configuration

Browse Source 
- instead of rotating logs daily and weekly, run the script every 15
  minutes
- merge weekly and daily configuration files
- rotate logs basing on two criteria: age (a week; only if file is
  bigger than 10M on master or 5MB on remote nodes) and size (rotate the file
  always if its size exceedes 100M on master or 20M on remote nodes, regardless
  its age)
- use dateformat to make rotated logs' filenames more readable;
  as logrotate supports only year, month, day and unix epoch, use them
  all
- fix a lot of lint complaints, including aligments, curly braces and
  proper quoting; give logrotate related files more intuitive names
- dump.log has been renamed to shotgun.log (LP#1318516)
- check if there is no logrotate instances running before executing
  cronjob

DocImpact: ops guide

Change-Id: Ie7040ebcdc1a3329ebb7507807e5c27f0c87b484
Closes-Bug: 1382515
Closes-Bug: 1446790
This commit is contained in:
Bartłomiej Piotrowski 2015-04-17 15:27:29 +02:00
parent 9b26cb98e7
commit eb3e29caec
13 changed files with 210 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
deployment/puppet/anacron/files/logrotate-hourly → deployment/puppet/anacron/files/logrotate
View File

@ -4,9 +4,15 @@
# Due to bug in logrotate, it always returns 0. Use grep for detect errors;
# exit code 1 is considered a success as no errors were found.
nice ionice -c3 /usr/sbin/logrotate /etc/logrotate.d/20-fuel*.conf >& /tmp/logrotate && grep -q error /tmp/logrotate
if ! pgrep logrotate &>/dev/null; then
nice ionice -c3 /usr/sbin/logrotate /etc/logrotate.d/fuel.conf >& /tmp/logrotate
grep -q error /tmp/logrotate
EXITVALUE=$?
else
/usr/bin/logger -t logrotate "WARNING another logrotate instance is already running, exiting"
exit 1
fi
EXITVALUE=$?
if [ $EXITVALUE != 1 ]; then
/usr/bin/logger -t logrotate "ALERT exited abnormally with [$EXITVALUE] (1 was expected)"
exit 1

12
deployment/puppet/anacron/files/logrotate-hourly-ubuntu → deployment/puppet/anacron/files/logrotate-ubuntu
View File

@ -10,10 +10,18 @@ do
done >> status.clean
mv status.clean status
test -x /usr/sbin/logrotate || exit 0
nice ionice -c3 /usr/sbin/logrotate /etc/logrotate.d/20-fuel.conf
EXITVALUE=$?
if ! pgrep logrotate &>/dev/null; then
nice ionice -c3 /usr/sbin/logrotate /etc/logrotate.d/fuel.conf
EXITVALUE=$?
else
/usr/bin/logger -t logrotate "WARNING another logrotate instance is already running, exiting"
exit 1
fi
if [ $EXITVALUE != 0 ]; then
/usr/bin/logger -t logrotate "ALERT exited abnormally with [$EXITVALUE] (0 was expected)"
fi
exit $EXITVALUE

52
deployment/puppet/anacron/manifests/config.pp
View File

@ -11,52 +11,70 @@ class anacron::config {
mode => '0644',
}
case $::operatingsystem {
/(?i)(centos|redhat)/: {
# assumes package cronie-anacron were installed at BM
case $::osfamily {
'RedHat': {
# assume cronie-anacron is installed
file { '/etc/anacrontab':
source => 'puppet:///modules/anacron/anacrontab',
}
file { '/etc/cron.hourly/':
ensure => directory,
mode => '0755',
owner => 'root',
mode => '0755',
owner => 'root',
}
file { '/etc/cron.d/':
ensure => directory,
mode => '0755',
owner => 'root',
mode => '0755',
owner => 'root',
}
file { '/etc/cron.d/0hourly':
source => 'puppet:///modules/anacron/0hourly',
}
file { '/etc/cron.hourly/logrotate':
file { '/etc/cron.d/fuel-logrotate':
mode => '0755',
source => 'puppet:///modules/anacron/logrotate-hourly',
source => 'puppet:///modules/anacron/logrotate',
}
file { '/etc/cron.hourly/0anacron':
mode => '0755',
source => 'puppet:///modules/anacron/0anacron-hourly',
}
}
'Debian': {
# assume anacron is installed
/(?i)(debian|ubuntu)/: {
# assumes package anacron were installed at BM
file { '/etc/anacrontab':
source => 'puppet:///modules/anacron/anacrontab-ubuntu',
}
file { '/etc/cron.d/anacron':
source => 'puppet:///modules/anacron/anacron-ubuntu',
}
file { '/etc/cron.hourly/logrotate':
file { '/etc/cron.d/fuel-logrotate':
mode => '0755',
source => 'puppet:///modules/anacron/logrotate-hourly-ubuntu',
source => 'puppet:///modules/anacron/logrotate-ubuntu',
}
}
}
if $::anacron::debug {
file { '/etc/cron.d/logrotate-debug':
source => 'puppet:///modules/anacron/logrotate-debug'
default: {
fail("Unsupported platform: ${::osfamily}/${::operatingsystem}")
}
}
if $::anacron::debug {
file { '/etc/cron.d/logrotate-debug':
source => 'puppet:///modules/anacron/logrotate-debug'
}
}
cron { 'fuel-logrotate':
command => '/etc/cron.d/fuel-logrotate',
user => 'root',
minute => '*/15',
}
}

9
deployment/puppet/nailgun/examples/host-only.pp
View File

@ -56,10 +56,11 @@ class { "docker":
}
class { 'openstack::logrotate':
role => 'server',
rotation => 'weekly',
keep => '4',
limitsize => '100M',
role => 'server',
rotation => 'weekly',
keep => '4',
minsize => '10M',
maxsize => '100M',
}
class { 'nailgun::client':

11
deployment/puppet/nailgun/examples/rsyslog-only.pp
View File

@ -27,9 +27,10 @@ file { $logconf :
notify => Class["::rsyslog::service"],
}
class {"::openstack::logrotate":
role => 'server',
rotation => 'weekly',
keep => '4',
limitsize => '100M',
class { '::openstack::logrotate':
role => 'server',
rotation => 'weekly',
keep => '4',
minsize => '10M',
maxsize => '20M',
}

6
deployment/puppet/nailgun/manifests/init.pp
View File

@ -98,15 +98,15 @@ class nailgun(
Class["nailgun::nginx-nailgun"],
],
}
class {openstack::logging:
class { 'openstack::logging':
role => 'server',
log_remote => false,
log_local => true,
log_auth_local => true,
rotation => 'weekly',
keep => '4',
# should be > 30M
limitsize => '100M',
minsize => '5M',
maxsize => '20M',
port => '514',
# listen both TCP and UDP
proto => 'both',

39
deployment/puppet/openstack/manifests/logging.pp
View File

@ -5,7 +5,8 @@
# [log_local], [log_auth_local] local & auth logging. Can be used with remote logging.
# [rotation] logrotate option for rotation period - daily, weekly, monthly, yearly.
# [keep] logrotate option for number or rotated log files to keep.
# [limitsize] logrotate option for log files would be rotated, if exceeded.
# [minsize] rotate log files periodically only if bigger than this value
# [maxsize] force rotate if this value has been exceeded
# [rservers] array of hashes which represents remote logging servers for client role.
# [port] port to use by server role for remote logging.
# [proto] tcp/udp/both proto(s) for remote log server role.
@ -18,22 +19,23 @@
# (rabbit does not support syslog, imfile is used for log capturing)
#
class openstack::logging (
$role = 'client',
$log_remote = true,
$log_local = false,
$log_auth_local = false,
$rotation = 'daily',
$keep = '7',
$limitsize = '300M',
$rservers = [{'remote_type'=>'udp', 'server'=>'master', 'port'=>'514'},],
$port = '514',
$proto = 'udp',
$show_timezone = false,
$virtual = false,
$rabbit_log_level = 'NOTICE',
$production = 'prod',
$escapenewline = false,
$debug = false,
$role = 'client',
$log_remote = true,
$log_local = false,
$log_auth_local = false,
$rotation = 'daily',
$keep = '7',
$minsize = '10M',
$maxsize = '100M',
$rservers = [{'remote_type'=>'udp', 'server'=>'master', 'port'=>'514'},],
$port = '514',
$proto = 'udp',
$show_timezone = false,
$virtual = false,
$rabbit_log_level = 'NOTICE',
$production = 'prod',
$escapenewline = false,
$debug = false,
) {
validate_re($proto, 'tcp|udp|both')
@ -260,7 +262,8 @@ class openstack::logging (
role => $role,
rotation => $rotation,
keep => $keep,
limitsize => $limitsize,
minsize => $minsize,
maxsize => $maxsize,
debug => $debug,
}

81
deployment/puppet/openstack/manifests/logrotate.pp
View File

@ -1,66 +1,45 @@
#
class openstack::logrotate (
$role = 'client',
$rotation = 'daily',
$keep = '7',
$limitsize = '300M',
$debug = false,
$role = 'client',
$rotation = 'weekly',
$keep = '4',
$minsize = '30M',
$maxsize = '100M',
$debug = false,
) {
validate_re($rotation, 'daily|weekly|monthly')
$logrotatefile = '/etc/logrotate.d/fuel'
if $role == 'server' {
# configure logs rotation both for host OS and docker containers of rsylog server role
# This file is used for daily/weekly/monthly log rotations
file { "/etc/logrotate.d/10-fuel-docker.conf":
owner => 'root',
group => 'root',
mode => '0644',
content => template("openstack/10-fuel-docker.conf.erb"),
# Configure log rotation for master node and docker containers
file { $logrotatefile:
owner => 'root',
group => 'root',
mode => '0644',
content => template('openstack/10-fuel-docker.conf.erb'),
}
# This file is used for hourly log rotations by (ana)cron
file { "/etc/logrotate.d/20-fuel-docker.conf":
owner => 'root',
group => 'root',
mode => '0644',
content => template("openstack/20-fuel-docker.conf.erb"),
}
$logrotatefile = '/etc/logrotate.d/20-fuel-docker.conf'
} else {
# configure logrotation for rsylog client role
# This file is used for daily/weekly/monthly log rotations
file { "/etc/logrotate.d/10-fuel.conf":
owner => 'root',
group => 'root',
mode => '0644',
content => template("openstack/10-fuel.conf.erb"),
# Configure log rotation for other nodes
file { $logrotatefile:
owner => 'root',
group => 'root',
mode => '0644',
content => template('openstack/10-fuel.conf.erb'),
}
# This file is used for hourly log rotations by (ana)cron
file { "/etc/logrotate.d/20-fuel.conf":
owner => 'root',
group => 'root',
mode => '0644',
content => template("openstack/20-fuel.conf.erb"),
}
$logrotatefile = '/etc/logrotate.d/20-fuel.conf'
}
# Configure (ana)cron for fuel custom hourly logrotations
class { '::anacron':
# Configure (ana)cron for fuel custom hourly logrotations
class { '::anacron':
debug => $debug,
}
case $osfamily {
'RedHat': {
# Due to bug existing, logrotate always returns 0. Use grep for detect errors:
# would return 1 (considered as normal result), if logrotate returns no errors, return 0, if any.
exec {'logrotate_check':
path => ["/usr/bin", "/usr/sbin", "/sbin", "/bin"],
command => "logrotate $logrotatefile >& /tmp/logrotate && grep -q error /tmp/logrotate",
returns => 1,
require => File[$logrotatefile],
}
if $::osfamily == 'RedHat' {
# Due to bug in logrotate, it always returns 0. Use grep to detect errors
# in output; exit code 1 is considered success as no errors were emitted.
exec {'logrotate_check':
path => ['/usr/bin', '/usr/sbin', '/sbin', '/bin'],
command => "logrotate ${logrotatefile} >& /tmp/logrotate && grep -q error /tmp/logrotate",
returns => 1,
require => File[$logrotatefile],
}
}
}
}

85
deployment/puppet/openstack/templates/10-fuel-docker.conf.erb
View File

@ -1,62 +1,73 @@
# managed by puppet
"/var/log/kern.log"
"/var/log/debug"
"/var/log/syslog"
"/var/log/daemon.log"
"/var/log/auth.log"
"/var/log/astute/*.log"
"/var/log/audit/audit.log"
"/var/log/user.log"
"/var/log/mail.log"
"/var/log/auth.log"
"/var/log/cobbler/*.log"
"/var/log/cron.log"
"/var/log/daemon.log"
"/var/log/debug"
"/var/log/docker"
"/var/log/docker-*.log"
"/var/log/lxc/*.log"
"/var/log/fuelmenu.log"
"/var/log/nailgun-agent.log"
"/var/log/nailgun/*.log"
"/var/log/astute/*.log"
"/var/log/ostf*.log"
"/var/log/httpd/*"
"/var/log/nginx/*.log"
"/var/log/rabbitmq/*"
"/var/log/cobbler/*.log"
"/var/log/dump.log"
"/var/log/remote/[1-9]*/*.log"
"/var/log/remote/[1-9]*/*/*.log"
"/var/log/docker-logs/dnsmasq.log"
"/var/log/docker-logs/astute/*.log"
"/var/log/docker-logs/cobbler/*.log"
"/var/log/docker-logs/ostf/*.log"
"/var/log/docker-logs/dnsmasq.log"
"/var/log/docker-logs/httpd/*.log"
"/var/log/docker-logs/nginx/*.log"
"/var/log/docker-logs/rabbitmq/*.log"
"/var/log/docker-logs/nailgun/*.log"
"/var/log/docker-logs/dump.log"
"/var/log/docker-logs/mcollective.log"
"/var/log/docker-logs/messages"
"/var/log/docker-logs/nailgun/*.log"
"/var/log/docker-logs/nginx/*.log"
"/var/log/docker-logs/ostf*.log"
"/var/log/docker-logs/ostf/*.log"
"/var/log/docker-logs/rabbitmq/*.log"
"/var/log/docker-logs/rsync.log"
"/var/log/docker-logs/shotgun.log"
"/var/log/fuelmenu.log"
"/var/log/httpd/*"
"/var/log/kern.log"
"/var/log/lxc/*.log"
"/var/log/mail.log"
"/var/log/nailgun-agent.log"
"/var/log/nailgun/*.log"
"/var/log/nginx/*.log"
"/var/log/ostf*.log"
"/var/log/rabbitmq/*"
"/var/log/remote/[1-9]*/*.log"
"/var/log/remote/[1-9]*/*/*.log"
"/var/log/shotgun.log"
"/var/log/syslog"
"/var/log/user.log"
{
# This file is used for daily log rotations, do not use size options here
# pass whole regexp to {pre,post}rotate scripts
sharedscripts
<%= @rotation %>
# rotate only if 30M size or bigger
minsize 30M
# truncate file, do not delete & recreate
copytruncate
# keep logs for <%= @keep %> rotations
rotate <%= @keep %>
# compression will be postponed to the next rotation, if uncommented
# delaycompress
# compress rotated files with gzip
compress
# ignore missing files
missingok
# do not rotate empty files
notifempty
# logrotate allows to use only year, month, day and unix epoch
dateformat -%Y%m%d-%s
# number of rotated files to keep
rotate <%= @keep %>
# do not rotate files unless both size and time conditions are met
<%= @rotation %>
minsize <%= @minsize %>
# force rotate if filesize exceeded <%= @maxsize %>
maxsize <%= @maxsize %>
postrotate
# issue HUPing for all rsyslog pids including ones in the docker containers, if any.
/usr/bin/pkill -HUP rsyslogd 2> /dev/null 2> /dev/null || true
reload rsyslog >/dev/null 2>&1 || true
# send SIGHUP to all rsyslog instances, including ones in containers
/usr/bin/pkill -HUP rsyslogd 2>/dev/null || true
endscript
}

71
deployment/puppet/openstack/templates/10-fuel.conf.erb
View File

@ -1,48 +1,61 @@
# managed by puppet
"/var/log/nailgun-agent.log"
"/var/log/*-all.log"
"/var/log/corosync.log"
"/var/log/kern.log"
"/var/log/debug"
"/var/log/syslog"
"/var/log/sudo.log"
"/var/log/daemon.log"
"/var/log/auth.log"
"/var/log/audit/audit.log"
"/var/log/user.log"
"/var/log/mail.log"
"/var/log/cron.log"
"/var/log/dashboard.log"
"/var/log/pacemaker.log"
"/var/log/ceilometer/*.log"
"/var/log/cinder/*.log"
"/var/log/glance/*.log"
"/var/log/heat/*.log"
"/var/log/keystone/*.log"
"/var/log/murano/*.log"
"/var/log/neutron/*.log"
"/var/log/nova/*.log"
"/var/log/keystone/*.log"
"/var/log/glance/*.log"
"/var/log/cinder/*.log"
"/var/log/sahara/*.log"
"/var/log/*-all.log"
"/var/log/auth.log"
"/var/log/corosync.log"
"/var/log/cron.log"
"/var/log/daemon.log"
"/var/log/dashboard.log"
"/var/log/debug"
"/var/log/kern.log"
"/var/log/mail.log"
"/var/log/messages"
"/var/log/mongod.log"
"/var/log/mysqld.log"
"/var/log/murano/*.log"
"/var/log/heat/*.log"
"/var/log/sahara/*.log"
"/var/log/ceilometer/*.log"
"/var/log/nailgun-agent.log"
"/var/log/pacemaker.log"
"/var/log/sudo.log"
"/var/log/syslog"
"/var/log/user.log"
{
# This file is used for daily log rotations, do not use size options here
# pass whole regexp to {pre,post}rotate scripts
sharedscripts
<%= @rotation %>
# rotate only if 30M size or bigger
minsize 30M
# truncate file, do not delete & recreate
copytruncate
# keep logs for <%= @keep %> rotations
rotate <%= @keep %>
# compression will be postponed to the next rotation, if uncommented
# delaycompress
# compress rotated files with gzip
compress
# ignore missing files
missingok
# do not rotate empty files
notifempty
# logrotate allows to use only year, month, day and unix epoch
dateformat -%Y%m%d-%s
# number of rotated files to keep
rotate <%= @keep %>
# do not rotate files unless both size and time conditions are met
<%= @rotation %>
minsize <%= @minsize %>
# force rotate if filesize exceeded <%= @maxsize %>
maxsize <%= @maxsize %>
postrotate
/bin/kill -HUP `cat /var/run/syslogd.pid 2> /dev/null` 2> /dev/null || true
reload rsyslog >/dev/null 2>&1 || true

65
deployment/puppet/openstack/templates/20-fuel-docker.conf.erb
View File

@ -1,65 +0,0 @@
# managed by puppet
"/var/log/kern.log"
"/var/log/debug"
"/var/log/syslog"
"/var/log/daemon.log"
"/var/log/auth.log"
"/var/log/audit/audit.log"
"/var/log/user.log"
"/var/log/mail.log"
"/var/log/cron.log"
"/var/log/docker"
"/var/log/docker-*.log"
"/var/log/lxc/*.log"
"/var/log/fuelmenu.log"
"/var/log/nailgun-agent.log"
"/var/log/nailgun/*.log"
"/var/log/astute/*.log"
"/var/log/ostf*.log"
"/var/log/httpd/*"
"/var/log/nginx/*.log"
"/var/log/rabbitmq/*"
"/var/log/cobbler/*.log"
"/var/log/dump.log"
"/var/log/remote/[1-9]*/*.log"
"/var/log/remote/[1-9]*/*/*.log"
"/var/log/docker-logs/dnsmasq.log"
"/var/log/docker-logs/astute/*.log"
"/var/log/docker-logs/cobbler/*.log"
"/var/log/docker-logs/ostf/*.log"
"/var/log/docker-logs/httpd/*.log"
"/var/log/docker-logs/nginx/*.log"
"/var/log/docker-logs/rabbitmq/*.log"
"/var/log/docker-logs/nailgun/*.log"
"/var/log/docker-logs/dump.log"
"/var/log/docker-logs/mcollective.log"
"/var/log/docker-logs/messages"
"/var/log/docker-logs/ostf*.log"
"/var/log/docker-logs/rsync.log"
{
# This file is used for hourly log rotations, use (min)size options here
sharedscripts
# truncate file, do not delete & recreate
copytruncate
# rotate only if 30M size or bigger
minsize 30M
# also rotate if <%= @limitsize %> size have exceeded, should be size > minsize
size <%= @limitsize %>
# keep logs for <%= @keep %> rotations
rotate <%= @keep %>
# compression will be postponed to the next rotation, if uncommented
# delaycompress
compress
# delaycompress
# ignore missing files
missingok
# do not rotate empty files
notifempty
postrotate
# issue HUPing for all rsyslog pids including ones in the docker containers, if any.
/usr/bin/pkill -HUP rsyslogd 2> /dev/null 2> /dev/null || true
reload rsyslog >/dev/null 2>&1 || true
endscript
}

54
deployment/puppet/openstack/templates/20-fuel.conf.erb
View File

@ -1,54 +0,0 @@
# managed by puppet
"/var/log/nailgun-agent.log"
"/var/log/*-all.log"
"/var/log/corosync.log"
"/var/log/kern.log"
"/var/log/debug"
"/var/log/syslog"
"/var/log/sudo.log"
"/var/log/daemon.log"
"/var/log/auth.log"
"/var/log/audit/audit.log"
"/var/log/user.log"
"/var/log/mail.log"
"/var/log/cron.log"
"/var/log/dashboard.log"
"/var/log/pacemaker.log"
"/var/log/neutron/*.log"
"/var/log/nova/*.log"
"/var/log/keystone/*.log"
"/var/log/glance/*.log"
"/var/log/cinder/*.log"
"/var/log/mongod.log"
"/var/log/mysqld.log"
"/var/log/murano/*.log"
"/var/log/heat/*.log"
"/var/log/sahara/*.log"
"/var/log/ceilometer/*.log"
"/var/log/messages"
{
# This file is used for hourly log rotations, use (min)size options here
sharedscripts
# truncate file, do not delete & recreate
copytruncate
# rotate only if 30M size or bigger
minsize 30M
# also rotate if <%= @limitsize %> size have exceeded, should be size > minsize
size <%= @limitsize %>
# keep logs for <%= @keep %> rotations
rotate <%= @keep %>
# compression will be postponed to the next rotation, if uncommented
# delaycompress
compress
# delaycompress
# ignore missing files
missingok
# do not rotate empty files
notifempty
postrotate
/bin/kill -HUP `cat /var/run/syslogd.pid 2> /dev/null` 2> /dev/null || true
reload rsyslog >/dev/null 2>&1 || true
endscript
}

4
deployment/puppet/osnailyfacter/modular/logging/logging.pp
View File

@ -39,8 +39,8 @@ if $use_syslog {
# force rotate if 300M size have exceeded
rotation => 'weekly',
keep => '4',
# should be > 30M
limitsize => '300M',
minsize => '10M',
maxsize => '100M',
# remote servers to send logs to
rservers => $rservers,
# should be true, if client is running at virtual node