Rotate hekad logs every 30 minutes if necessary

This change rotates the hekad logs more frequently. It also rotates the
log file when it reaches a certain size.

Fixes-bug: #1561603

Change-Id: Ic08831b8abadd0e1f846e0f401dc74b15dd46b3c

deployment_scripts/puppet/modules/heka/manifests/init.pp

@@ -148,9 +148,34 @@ class heka (
     require => [User[$heka_user], Package['heka']],
   }
 
-  file { "/etc/logrotate.d/${service_name}":
+  $logrotate_conf = "/etc/logrotate_${service_name}.conf"
+  file { $logrotate_conf:
     ensure  => present,
     content => template('heka/logrotate.conf.erb'),
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    require => Package['heka'],
+  }
+
+  $logrotate_bin = "/usr/local/bin/logrotate_${service_name}"
+  file { $logrotate_bin:
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    content => template('heka/logrotate.cron.erb'),
+    require => File[$logrotate_conf],
+  }
+
+  cron { "${service_name} logrotate":
+    ensure   => present,
+    command  => $logrotate_bin,
+    minute   => '*/30',
+    hour     => '*',
+    month    => '*',
+    monthday => '*',
+    require  => File[$logrotate_bin],
   }
 
   file { $hekad_wrapper:

deployment_scripts/puppet/modules/heka/templates/logrotate.conf.erb

@@ -1,7 +1,22 @@
+# managed by puppet
 <%= @log_file %> {
-    daily
-    missingok
     # Heka cannot be told to close its log file and re-open it so we need to
     # use the copytruncate option
     copytruncate
+    compress
+    delaycompress
+    missingok
+    notifempty
+    # logrotate allows to use only year, month, day and unix epoch
+    dateext
+    dateformat -%Y%m%d-%s
+    # number of rotated files to keep
+    rotate 10
+    # do not rotate files unless both size and time conditions are met
+    hourly
+    minsize 20M
+    # force rotate if filesize exceeded 100M
+    maxsize 100M
+    # this must map the /var/log directory group ownership
+    su root syslog
 }

deployment_scripts/puppet/modules/heka/templates/logrotate.cron.erb

@@ -0,0 +1,54 @@
+#!/bin/bash
+# managed by puppet
+
+test -x /usr/sbin/logrotate || exit 0
+
+LOCK_FILE="/var/lock/logrotate.<%= @service_name %>.lock"
+
+lock() {
+    exec 903>$LOCK_FILE
+    flock -n 903 && return 0 || return 1
+}
+
+unlock() {
+    flock -u 903
+}
+
+fail() {
+    if [ -z "$1" ]
+    then
+        MESSAGE="WARNING logrotate failed, no reason provided"
+    else
+        MESSAGE=$1
+    fi
+    /usr/bin/logger -t logrotate "${MESSAGE}"
+    unlock
+    exit 1
+}
+
+lock || fail "WARNING <%= @service_name %> logrotate flock failed, exiting"
+
+
+TMP_FILE=$(/bin/mktemp)
+nice ionice -c3 /usr/sbin/logrotate <%= @logrotate_conf %> >& $TMP_FILE
+EXITVALUE=$?
+
+if [ -f /etc/redhat-release ] || [ -f /etc/centos-release ];
+then
+    # Due to bug in logrotate on centos/rhel, it always returns 0. Use grep for
+    # detect errors; exit code 1 is considered a success as no errors were
+    # found.
+    grep -q error $TMP_FILE
+    EXITVALUE=$?
+    EXPECTEDVALUE=1
+else
+    EXPECTEDVALUE=0
+fi
+rm $TMP_FILE
+
+if [ "${EXITVALUE}" != "${EXPECTEDVALUE}" ]; then
+    fail "ALERT exited abnormally with [${EXITVALUE}] (${EXPECTEDVALUE} was expected)"
+fi
+
+unlock
+exit 0