Browse Source

Rotate hekad logs every 30 minutes if necessary

This change rotates the hekad logs more frequently. It also rotates the
log file when it reaches a certain size.

Fixes-bug: #1561603

Change-Id: Ic08831b8abadd0e1f846e0f401dc74b15dd46b3c
(cherry picked from commit b46fcb0417)
Swann Croiset 3 years ago
parent
commit
65ad559a9a

+ 26
- 1
deployment_scripts/puppet/modules/heka/manifests/init.pp View File

@@ -149,9 +149,34 @@ class heka (
149 149
     require => [User[$heka_user], Package['heka']],
150 150
   }
151 151
 
152
-  file { "/etc/logrotate.d/${service_name}":
152
+  $logrotate_conf = "/etc/logrotate_${service_name}.conf"
153
+  file { $logrotate_conf:
153 154
     ensure  => present,
154 155
     content => template('heka/logrotate.conf.erb'),
156
+    owner   => 'root',
157
+    group   => 'root',
158
+    mode    => '0644',
159
+    require => Package['heka'],
160
+  }
161
+
162
+  $logrotate_bin = "/usr/local/bin/logrotate_${service_name}"
163
+  file { $logrotate_bin:
164
+    ensure  => present,
165
+    owner   => 'root',
166
+    group   => 'root',
167
+    mode    => '0755',
168
+    content => template('heka/logrotate.cron.erb'),
169
+    require => File[$logrotate_conf],
170
+  }
171
+
172
+  cron { "${service_name} logrotate":
173
+    ensure   => present,
174
+    command  => $logrotate_bin,
175
+    minute   => '*/30',
176
+    hour     => '*',
177
+    month    => '*',
178
+    monthday => '*',
179
+    require  => File[$logrotate_bin],
155 180
   }
156 181
 
157 182
   file { $hekad_wrapper:

+ 17
- 2
deployment_scripts/puppet/modules/heka/templates/logrotate.conf.erb View File

@@ -1,7 +1,22 @@
1
+# managed by puppet
1 2
 <%= @log_file %> {
2
-    daily
3
-    missingok
4 3
     # Heka cannot be told to close its log file and re-open it so we need to
5 4
     # use the copytruncate option
6 5
     copytruncate
6
+    compress
7
+    delaycompress
8
+    missingok
9
+    notifempty
10
+    # logrotate allows to use only year, month, day and unix epoch
11
+    dateext
12
+    dateformat -%Y%m%d-%s
13
+    # number of rotated files to keep
14
+    rotate 10
15
+    # do not rotate files unless both size and time conditions are met
16
+    hourly
17
+    minsize 20M
18
+    # force rotate if filesize exceeded 100M
19
+    maxsize 100M
20
+    # this must map the /var/log directory group ownership
21
+    su root syslog
7 22
 }

+ 54
- 0
deployment_scripts/puppet/modules/heka/templates/logrotate.cron.erb View File

@@ -0,0 +1,54 @@
1
+#!/bin/bash
2
+# managed by puppet
3
+
4
+test -x /usr/sbin/logrotate || exit 0
5
+
6
+LOCK_FILE="/var/lock/logrotate.<%= @service_name %>.lock"
7
+
8
+lock() {
9
+    exec 903>$LOCK_FILE
10
+    flock -n 903 && return 0 || return 1
11
+}
12
+
13
+unlock() {
14
+    flock -u 903
15
+}
16
+
17
+fail() {
18
+    if [ -z "$1" ]
19
+    then
20
+        MESSAGE="WARNING logrotate failed, no reason provided"
21
+    else
22
+        MESSAGE=$1
23
+    fi
24
+    /usr/bin/logger -t logrotate "${MESSAGE}"
25
+    unlock
26
+    exit 1
27
+}
28
+
29
+lock || fail "WARNING <%= @service_name %> logrotate flock failed, exiting"
30
+
31
+
32
+TMP_FILE=$(/bin/mktemp)
33
+nice ionice -c3 /usr/sbin/logrotate <%= @logrotate_conf %> >& $TMP_FILE
34
+EXITVALUE=$?
35
+
36
+if [ -f /etc/redhat-release ] || [ -f /etc/centos-release ];
37
+then
38
+    # Due to bug in logrotate on centos/rhel, it always returns 0. Use grep for
39
+    # detect errors; exit code 1 is considered a success as no errors were
40
+    # found.
41
+    grep -q error $TMP_FILE
42
+    EXITVALUE=$?
43
+    EXPECTEDVALUE=1
44
+else
45
+    EXPECTEDVALUE=0
46
+fi
47
+rm $TMP_FILE
48
+
49
+if [ "${EXITVALUE}" != "${EXPECTEDVALUE}" ]; then
50
+    fail "ALERT exited abnormally with [${EXITVALUE}] (${EXPECTEDVALUE} was expected)"
51
+fi
52
+
53
+unlock
54
+exit 0

Loading…
Cancel
Save