From b685f54711075d9c5b665f6d5c5be00ca6e5369a Mon Sep 17 00:00:00 2001 From: Alex Schultz Date: Fri, 4 Sep 2015 14:35:39 -0500 Subject: [PATCH] Fix logrotate to prevent downing all apache svcs This change introduces a random delay between 0 and 5 minutes for logrotate as part of the pre-rotate process for apache2. This sleep is an attempt to prevent all apache services from being restarted at the exact same time for a cluster. Additionally we have found that soemtimes apache will crash with mod_wsgi so this change updates the two places where we use a graceful restart to also include a restart of apache if the graceful is unsuccessful. Change-Id: I0585835ca67e4137e39056b51965c922bb0118cd Closes-Bug: 1491576 --- .../puppet/osnailyfacter/manifests/apache.pp | 61 ++++++++++++++++++- .../templates/apache2.logrotate.erb | 21 +++++++ .../templates/apache2.prerotate.erb | 7 +++ .../tweaks/manifests/apache_wrappers.pp | 5 +- tests/noop/spec/hosts/apache/apache_spec.rb | 18 ++++++ .../spec/hosts/api-proxy/api-proxy_spec.rb | 2 +- .../hosts/astute/service_token_off_spec.rb | 2 +- tests/noop/spec/hosts/ceph/radosgw_spec.rb | 2 +- tests/noop/spec/hosts/horizon/horizon_spec.rb | 2 +- .../noop/spec/hosts/keystone/keystone_spec.rb | 2 +- 10 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 deployment/puppet/osnailyfacter/templates/apache2.logrotate.erb create mode 100644 deployment/puppet/osnailyfacter/templates/apache2.prerotate.erb diff --git a/deployment/puppet/osnailyfacter/manifests/apache.pp b/deployment/puppet/osnailyfacter/manifests/apache.pp index c98e98098b..a27c807ba2 100644 --- a/deployment/puppet/osnailyfacter/manifests/apache.pp +++ b/deployment/puppet/osnailyfacter/manifests/apache.pp @@ -1,7 +1,27 @@ -# Configure apache and listen ports +# == Class: osnailyfacter::apache +# +# Configure apache and listen ports. This class also manages the apache2 +# logrotate configuration. +# +# === Parameters +# +# [*purge_configs*] +# (optional) Boolean flag to indicate if we should purge all the apache +# configs unless explicitly managed via puppet. +# Defaults to false +# +# [*listen_ports*] +# (optional) The ports to listen on for apache +# Defaults to '80' +# +# [*logrotate_rotate*] +# (optional) The number of times to be rotated before being removed. +# Defaults to '52' +# class osnailyfacter::apache ( - $purge_configs = false, - $listen_ports = '80', + $purge_configs = false, + $listen_ports = '80', + $logrotate_rotate = '52', ) { define apache_port { @@ -20,4 +40,39 @@ class osnailyfacter::apache ( } apache_port { $listen_ports: } + + # we need to override the logrotate file provided by apache to work around + # wsgi issues on the restart caused by logrotate. + # LP#1491576 and https://github.com/GrahamDumpleton/mod_wsgi/issues/81 + file { '/etc/logrotate.d/apache2': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0644', + content => template('osnailyfacter/apache2.logrotate.erb'), + require => Package['httpd'] + } + + # This will randomly rotate the array of delays based on hostname to allow + # for an idempotent delay to be applied. This will introduce a delay between + # 0 and 5 minutes to the logrotate process. + $delay = fqdn_rotate([0,1,2,3,4,5]) + + # Convert delay into seconds for the prerotation script + $apache2_logrotate_delay = $delay[0] * 60 + + file { '/etc/logrotate.d/httpd-prerotate': + ensure => 'directory', + owner => 'root', + group => 'root', + mode => '0755', + } + + file { '/etc/logrotate.d/httpd-prerotate/apache2': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0755', + content => template('osnailyfacter/apache2.prerotate.erb'), + } } diff --git a/deployment/puppet/osnailyfacter/templates/apache2.logrotate.erb b/deployment/puppet/osnailyfacter/templates/apache2.logrotate.erb new file mode 100644 index 0000000000..d0fe88819e --- /dev/null +++ b/deployment/puppet/osnailyfacter/templates/apache2.logrotate.erb @@ -0,0 +1,21 @@ +# This file managed via puppet +/var/log/apache2/*.log { + weekly + missingok + rotate <%= @logrotate_rotate %> + compress + delaycompress + notifempty + create 640 root adm + sharedscripts + postrotate + if /etc/init.d/apache2 status > /dev/null ; then \ + (/usr/sbin/apachectl graceful) || (/usr/sbin/apachectl restart) + fi; + endscript + prerotate + if [ -d /etc/logrotate.d/httpd-prerotate ]; then \ + run-parts /etc/logrotate.d/httpd-prerotate; \ + fi; \ + endscript +} diff --git a/deployment/puppet/osnailyfacter/templates/apache2.prerotate.erb b/deployment/puppet/osnailyfacter/templates/apache2.prerotate.erb new file mode 100644 index 0000000000..62b5bdfea9 --- /dev/null +++ b/deployment/puppet/osnailyfacter/templates/apache2.prerotate.erb @@ -0,0 +1,7 @@ +#!/bin/sh +# This is a prerotate script for apache2 that will add a delay to the log +# rotation to spread out the apache2 restarts. The goal of this script is to +# stager the apache restarts to prevent all services from being down at the +# same time. LP#1491576 + +sleep <%=@apache2_logrotate_delay%> diff --git a/deployment/puppet/tweaks/manifests/apache_wrappers.pp b/deployment/puppet/tweaks/manifests/apache_wrappers.pp index a9cb5eb1c2..9206f6a357 100644 --- a/deployment/puppet/tweaks/manifests/apache_wrappers.pp +++ b/deployment/puppet/tweaks/manifests/apache_wrappers.pp @@ -8,8 +8,11 @@ class tweaks::apache_wrappers ( default => fail("Unsupported osfamily: ${::osfamily}"), } + # we try a graceful restart but will fall back to a restart if graceful fails + # as we have found that sometimes with mod_wsgi apache will crash on a + # graceful restart - https://github.com/GrahamDumpleton/mod_wsgi/issues/81 Service <| name == $service_name or title == $service_name |> { - restart => 'apachectl graceful', + restart => 'apachectl graceful || apachectl restart', hasrestart => true, } } diff --git a/tests/noop/spec/hosts/apache/apache_spec.rb b/tests/noop/spec/hosts/apache/apache_spec.rb index 3d29b5fe35..1559bb6500 100644 --- a/tests/noop/spec/hosts/apache/apache_spec.rb +++ b/tests/noop/spec/hosts/apache/apache_spec.rb @@ -25,6 +25,24 @@ describe manifest do ) } + it 'should contain apache2 logrotate overrides' do + should contain_file('/etc/logrotate.d/apache2').with( + :ensure => 'file', + :owner => 'root', + :group => 'root', + :mode => '0644').with_content(/rotate 52/) + should contain_file('/etc/logrotate.d/httpd-prerotate').with( + :ensure => 'directory', + :owner => 'root', + :group => 'root', + :mode => '0755') + should contain_file('/etc/logrotate.d/httpd-prerotate/apache2').with( + :ensure => 'file', + :owner => 'root', + :group => 'root', + :mode => '0755').with_content(/^sleep \d+/) + end + end test_ubuntu_and_centos manifest end diff --git a/tests/noop/spec/hosts/api-proxy/api-proxy_spec.rb b/tests/noop/spec/hosts/api-proxy/api-proxy_spec.rb index d1472f6bf0..8f04b22fa1 100644 --- a/tests/noop/spec/hosts/api-proxy/api-proxy_spec.rb +++ b/tests/noop/spec/hosts/api-proxy/api-proxy_spec.rb @@ -7,7 +7,7 @@ describe manifest do it { should contain_service('httpd').with( 'hasrestart' => true, - 'restart' => 'apachectl graceful', + 'restart' => 'apachectl graceful || apachectl restart' ) } end diff --git a/tests/noop/spec/hosts/astute/service_token_off_spec.rb b/tests/noop/spec/hosts/astute/service_token_off_spec.rb index 2758456caa..f2aec92fa9 100644 --- a/tests/noop/spec/hosts/astute/service_token_off_spec.rb +++ b/tests/noop/spec/hosts/astute/service_token_off_spec.rb @@ -20,7 +20,7 @@ describe manifest do :ensure => 'running', :name => service_name, :hasrestart => 'true', - :restart => 'apachectl graceful', + :restart => 'apachectl graceful || apachectl restart' ) end diff --git a/tests/noop/spec/hosts/ceph/radosgw_spec.rb b/tests/noop/spec/hosts/ceph/radosgw_spec.rb index 7fc5bbff43..974e767311 100644 --- a/tests/noop/spec/hosts/ceph/radosgw_spec.rb +++ b/tests/noop/spec/hosts/ceph/radosgw_spec.rb @@ -18,7 +18,7 @@ describe manifest do it { should contain_service('httpd').with( 'hasrestart' => true, - 'restart' => 'apachectl graceful', + 'restart' => 'apachectl graceful || apachectl restart', ) } diff --git a/tests/noop/spec/hosts/horizon/horizon_spec.rb b/tests/noop/spec/hosts/horizon/horizon_spec.rb index 4df99c3a8d..789c5448fc 100644 --- a/tests/noop/spec/hosts/horizon/horizon_spec.rb +++ b/tests/noop/spec/hosts/horizon/horizon_spec.rb @@ -45,7 +45,7 @@ describe manifest do it { should contain_service('httpd').with( 'hasrestart' => true, - 'restart' => 'apachectl graceful', + 'restart' => 'apachectl graceful || apachectl restart' ) } diff --git a/tests/noop/spec/hosts/keystone/keystone_spec.rb b/tests/noop/spec/hosts/keystone/keystone_spec.rb index 9e9180e826..00fb6eecf3 100644 --- a/tests/noop/spec/hosts/keystone/keystone_spec.rb +++ b/tests/noop/spec/hosts/keystone/keystone_spec.rb @@ -159,7 +159,7 @@ describe manifest do it { should contain_service('httpd').with( 'hasrestart' => true, - 'restart' => 'apachectl graceful', + 'restart' => 'apachectl graceful || apachectl restart' ) }