Fix logrotate to prevent downing all apache svcs

This change introduces a random delay between 0 and 5 minutes for
logrotate as part of the pre-rotate process for apache2. This sleep is
an attempt to prevent all apache services from being restarted
at the exact same time for a cluster. Additionally we have found that
soemtimes apache will crash with mod_wsgi so this change updates the two
places where we use a graceful restart to also include a restart of
apache if the graceful is unsuccessful.

Change-Id: I0585835ca67e4137e39056b51965c922bb0118cd
Closes-Bug: 1491576

deployment/puppet/osnailyfacter/manifests/apache.pp

@@ -1,7 +1,27 @@
-# Configure apache and listen ports
+# == Class: osnailyfacter::apache
+#
+#  Configure apache and listen ports. This class also manages the apache2
+#  logrotate configuration.
+#
+# === Parameters
+#
+# [*purge_configs*]
+#  (optional) Boolean flag to indicate if we should purge all the apache
+#  configs unless explicitly managed via puppet.
+#  Defaults to false
+#
+# [*listen_ports*]
+#  (optional) The ports to listen on for apache
+#  Defaults to '80'
+#
+# [*logrotate_rotate*]
+#  (optional) The number of times to be rotated before being removed.
+#  Defaults to '52'
+#
 class osnailyfacter::apache (
-  $purge_configs = false,
-  $listen_ports  = '80',
+  $purge_configs    = false,
+  $listen_ports     = '80',
+  $logrotate_rotate = '52',
 ) {
 
   define apache_port {
@@ -20,4 +40,39 @@ class osnailyfacter::apache (
   }
 
   apache_port { $listen_ports: }
+
+  # we need to override the logrotate file provided by apache to work around
+  # wsgi issues on the restart caused by logrotate.
+  # LP#1491576 and https://github.com/GrahamDumpleton/mod_wsgi/issues/81
+  file { '/etc/logrotate.d/apache2':
+    ensure  => 'file',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('osnailyfacter/apache2.logrotate.erb'),
+    require => Package['httpd']
+  }
+
+  # This will randomly rotate the array of delays based on hostname to allow
+  # for an idempotent delay to be applied. This will introduce a delay between
+  # 0 and 5 minutes to the logrotate process.
+  $delay = fqdn_rotate([0,1,2,3,4,5])
+
+  # Convert delay into seconds for the prerotation script
+  $apache2_logrotate_delay = $delay[0] * 60
+
+  file { '/etc/logrotate.d/httpd-prerotate':
+    ensure => 'directory',
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+
+  file { '/etc/logrotate.d/httpd-prerotate/apache2':
+    ensure  => 'file',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    content => template('osnailyfacter/apache2.prerotate.erb'),
+  }
 }

deployment/puppet/osnailyfacter/templates/apache2.logrotate.erb

@@ -0,0 +1,21 @@
+# This file managed via puppet
+/var/log/apache2/*.log {
+    weekly
+    missingok
+    rotate <%= @logrotate_rotate %>
+    compress
+    delaycompress
+    notifempty
+    create 640 root adm
+    sharedscripts
+    postrotate
+       if /etc/init.d/apache2 status > /dev/null ; then \
+           (/usr/sbin/apachectl graceful) || (/usr/sbin/apachectl restart)
+       fi;
+    endscript
+    prerotate
+        if [ -d /etc/logrotate.d/httpd-prerotate ]; then \
+            run-parts /etc/logrotate.d/httpd-prerotate; \
+        fi; \
+    endscript
+}

deployment/puppet/osnailyfacter/templates/apache2.prerotate.erb

@@ -0,0 +1,7 @@
+#!/bin/sh
+# This is a prerotate script for apache2 that will add a delay to the log
+# rotation to spread out the apache2 restarts. The goal of this script is to
+# stager the apache restarts to prevent all services from being down at the
+# same time. LP#1491576
+
+sleep <%=@apache2_logrotate_delay%>

deployment/puppet/tweaks/manifests/apache_wrappers.pp

@@ -8,8 +8,11 @@ class tweaks::apache_wrappers (
     default  => fail("Unsupported osfamily: ${::osfamily}"),
   }
 
+  # we try a graceful restart but will fall back to a restart if graceful fails
+  # as we have found that sometimes with mod_wsgi apache will crash on a
+  # graceful restart - https://github.com/GrahamDumpleton/mod_wsgi/issues/81
   Service <| name == $service_name or title == $service_name |> {
-    restart    => 'apachectl graceful',
+    restart    => 'apachectl graceful || apachectl restart',
     hasrestart => true,
   }
 }

tests/noop/spec/hosts/apache/apache_spec.rb

@@ -25,6 +25,24 @@ describe manifest do
       )
     }
 
+    it 'should contain apache2 logrotate overrides' do
+      should contain_file('/etc/logrotate.d/apache2').with(
+        :ensure => 'file',
+        :owner  => 'root',
+        :group  => 'root',
+        :mode   => '0644').with_content(/rotate 52/)
+      should contain_file('/etc/logrotate.d/httpd-prerotate').with(
+        :ensure => 'directory',
+        :owner  => 'root',
+        :group  => 'root',
+        :mode   => '0755')
+      should contain_file('/etc/logrotate.d/httpd-prerotate/apache2').with(
+        :ensure => 'file',
+        :owner  => 'root',
+        :group  => 'root',
+        :mode   => '0755').with_content(/^sleep \d+/)
+    end
+
   end
   test_ubuntu_and_centos manifest
 end

tests/noop/spec/hosts/api-proxy/api-proxy_spec.rb

@@ -7,7 +7,7 @@ describe manifest do
     it {
       should contain_service('httpd').with(
            'hasrestart' => true,
-           'restart'    => 'apachectl graceful',
+           'restart'    => 'apachectl graceful || apachectl restart'
       )
     }
   end

tests/noop/spec/hosts/astute/service_token_off_spec.rb

@@ -20,7 +20,7 @@ describe manifest do
           :ensure     => 'running',
           :name       => service_name,
           :hasrestart => 'true',
-          :restart    => 'apachectl graceful',
+          :restart    => 'apachectl graceful || apachectl restart'
         )
       end
 

tests/noop/spec/hosts/ceph/radosgw_spec.rb

@@ -18,7 +18,7 @@ describe manifest do
       it {
         should contain_service('httpd').with(
              'hasrestart' => true,
-             'restart'    => 'apachectl graceful',
+             'restart'    => 'apachectl graceful || apachectl restart',
         )
       }
 

tests/noop/spec/hosts/horizon/horizon_spec.rb

@@ -45,7 +45,7 @@ describe manifest do
     it {
       should contain_service('httpd').with(
            'hasrestart' => true,
-           'restart'    => 'apachectl graceful',
+           'restart'    => 'apachectl graceful || apachectl restart'
       )
     }
 

tests/noop/spec/hosts/keystone/keystone_spec.rb

@@ -159,7 +159,7 @@ describe manifest do
      it {
        should contain_service('httpd').with(
             'hasrestart' => true,
-            'restart'    => 'apachectl graceful',
+            'restart'    => 'apachectl graceful || apachectl restart'
        )
      }