From 285554ba88f80ff1c00ab31da4e12284831e94f1 Mon Sep 17 00:00:00 2001 From: Carlos Camacho Date: Wed, 21 Mar 2018 16:58:45 +0000 Subject: [PATCH] Add a retention policy (age) when purging shadow tables This submission changes the --all default parameter in the purge shadow tables cron to a '--before ' allowing to define a retention policy. Age name was used as puppet-cinder already use it so to have a consitancy across the projects. Also, it's added the option --all-cells causing the purge to be applied against all cell databases. Co-Authored-By: Sergii Golovatiuk Change-Id: I2dcf37417c36fb8b1bde207c60d22d580005715c --- manifests/cron/purge_shadow_tables.pp | 39 +++- ...for-retention-policy-ff352f29c129844f.yaml | 14 ++ .../nova_cron_archive_deleted_rows_spec.rb | 206 +++++++++--------- spec/classes/nova_cron_purge_shadow_tables.rb | 139 +++++++----- 4 files changed, 239 insertions(+), 159 deletions(-) create mode 100644 releasenotes/notes/change-purge-all-for-retention-policy-ff352f29c129844f.yaml diff --git a/manifests/cron/purge_shadow_tables.pp b/manifests/cron/purge_shadow_tables.pp index b4dd7e995..6ed8db528 100644 --- a/manifests/cron/purge_shadow_tables.pp +++ b/manifests/cron/purge_shadow_tables.pp @@ -23,7 +23,7 @@ # (optional) Defaults to '0'. # # [*hour*] -# (optional) Defaults to '12'. +# (optional) Defaults to '5'. # # [*monthday*] # (optional) Defaults to '*'. @@ -32,7 +32,7 @@ # (optional) Defaults to '*'. # # [*weekday*] -# (optional) Defaults to '6'. +# (optional) Defaults to '*'. # # [*user*] # (optional) User with access to nova files. @@ -43,20 +43,30 @@ # (optional) Path to file to which rows should be archived # Defaults to '/var/log/nova/nova-rowspurge.log'. # +# [*age*] +# (optional) Adds a retention policy when purging the shadow tables +# Defaults to 14. +# +# [*all_cells*] +# (optional) Adds --all-cells to the purge command +# Defaults to false. +# # [*verbose*] # (optional) Adds --verbose to the purge command # If specified, will print information about the purged rows. # class nova::cron::purge_shadow_tables ( - $minute = 0, - $hour = 12, - $monthday = '*', - $month = '*', - $weekday = '6', - $user = undef, - $destination = '/var/log/nova/nova-rowspurge.log', - $verbose = false, + $minute = 0, + $hour = 5, + $monthday = '*', + $month = '*', + $weekday = '*', + $user = undef, + $destination = '/var/log/nova/nova-rowspurge.log', + $age = 14, + $all_cells = false, + $verbose = false, ) { include ::nova::deps @@ -69,10 +79,17 @@ class nova::cron::purge_shadow_tables ( $verbose_real = '' } + if $all_cells { + $all_cells_real = '--all-cells' + } + else { + $all_cells_real = '' + } + $cron_cmd = 'nova-manage db purge' cron { 'nova-manage db purge': - command => "${cron_cmd} --all ${verbose_real} >>${destination} 2>&1", + command => "${cron_cmd} --before `date --date='today - ${age} days' +%D` ${verbose_real} ${all_cells_real} >>${destination} 2>&1", environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', user => pick($user, $::nova::params::nova_user), minute => $minute, diff --git a/releasenotes/notes/change-purge-all-for-retention-policy-ff352f29c129844f.yaml b/releasenotes/notes/change-purge-all-for-retention-policy-ff352f29c129844f.yaml new file mode 100644 index 000000000..818eb895d --- /dev/null +++ b/releasenotes/notes/change-purge-all-for-retention-policy-ff352f29c129844f.yaml @@ -0,0 +1,14 @@ +--- +features: + - | + This patch will add a new parameter (age) to define a + retention policy when purging the Nova shadow tables. + Also will add the parameter all_cells (defaulted to false) + to run the purge command over the cells tables. +fixes: + - | + Fix the default values to saner ones, this is because + operators might run both the archive and the purge cron + jobs. The defaults will make the purge job to run each + day at 5 hours but the retention policy parameter will + retain the data for 14 days. diff --git a/spec/classes/nova_cron_archive_deleted_rows_spec.rb b/spec/classes/nova_cron_archive_deleted_rows_spec.rb index 72dd34926..558d42a46 100644 --- a/spec/classes/nova_cron_archive_deleted_rows_spec.rb +++ b/spec/classes/nova_cron_archive_deleted_rows_spec.rb @@ -2,106 +2,118 @@ require 'spec_helper' describe 'nova::cron::archive_deleted_rows' do - let :facts do - OSDefaults.get_facts({ :osfamily => 'Debian' }) + shared_examples_for 'nova::cron::archive_deleted_rows' do + + let :params do + { :minute => 1, + :hour => 0, + :monthday => '*', + :month => '*', + :weekday => '*', + :max_rows => '100', + :user => 'nova', + :until_complete => false, + :destination => '/var/log/nova/nova-rowsflush.log' } + end + + context 'until_complete is false' do + it 'configures a cron without until_complete' do + is_expected.to contain_cron('nova-manage db archive_deleted_rows').with( + :command => "nova-manage db archive_deleted_rows --max_rows #{params[:max_rows]} >>#{params[:destination]} 2>&1", + :user => 'nova', + :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', + :user => params[:user], + :minute => params[:minute], + :hour => params[:hour], + :monthday => params[:monthday], + :month => params[:month], + :weekday => params[:weekday], + :require => 'Anchor[nova::dbsync::end]', + ) + end + end + + context 'until_complete is true' do + before :each do + params.merge!( + :until_complete => true, + ) + end + + it 'configures a cron with until_complete' do + is_expected.to contain_cron('nova-manage db archive_deleted_rows').with( + :command => "nova-manage db archive_deleted_rows --max_rows #{params[:max_rows]} --until-complete >>#{params[:destination]} 2>&1", + :user => 'nova', + :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', + :user => params[:user], + :minute => params[:minute], + :hour => params[:hour], + :monthday => params[:monthday], + :month => params[:month], + :weekday => params[:weekday], + :require => 'Anchor[nova::dbsync::end]', + ) + end + end + + context 'purge is true' do + before :each do + params.merge!( + :purge => true, + ) + end + + it 'configures a cron with purge' do + is_expected.to contain_cron('nova-manage db archive_deleted_rows').with( + :command => "nova-manage db archive_deleted_rows --purge --max_rows #{params[:max_rows]} >>#{params[:destination]} 2>&1", + :user => 'nova', + :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', + :user => params[:user], + :minute => params[:minute], + :hour => params[:hour], + :monthday => params[:monthday], + :month => params[:month], + :weekday => params[:weekday], + :require => 'Anchor[nova::dbsync::end]', + ) + end + end + + context 'full purge' do + before :each do + params.merge!( + :purge => true, + :until_complete => true, + ) + end + + it 'configures a cron with all purge params' do + is_expected.to contain_cron('nova-manage db archive_deleted_rows').with( + :command => "nova-manage db archive_deleted_rows --purge --max_rows #{params[:max_rows]} --until-complete >>#{params[:destination]} 2>&1", + :user => 'nova', + :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', + :user => params[:user], + :minute => params[:minute], + :hour => params[:hour], + :monthday => params[:monthday], + :month => params[:month], + :weekday => params[:weekday], + :require => 'Anchor[nova::dbsync::end]', + ) + end + end + end - let :params do - { :minute => 1, - :hour => 0, - :monthday => '*', - :month => '*', - :weekday => '*', - :max_rows => '100', - :user => 'nova', - :until_complete => false, - :destination => '/var/log/nova/nova-rowsflush.log' } - end - - context 'until_complete is false' do - it 'configures a cron without until_complete' do - is_expected.to contain_cron('nova-manage db archive_deleted_rows').with( - :command => "nova-manage db archive_deleted_rows --max_rows #{params[:max_rows]} >>#{params[:destination]} 2>&1", - :user => 'nova', - :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', - :user => params[:user], - :minute => params[:minute], - :hour => params[:hour], - :monthday => params[:monthday], - :month => params[:month], - :weekday => params[:weekday], - :require => 'Anchor[nova::dbsync::end]', - ) + on_supported_os({ + :supported_os => OSDefaults.get_supported_os + }).each do |os,facts| + context "on #{os}" do + let (:facts) do + facts.merge!(OSDefaults.get_facts()) + end + it_configures 'nova::cron::archive_deleted_rows' end end - context 'until_complete is true' do - before :each do - params.merge!( - :until_complete => true, - ) - end - - it 'configures a cron with until_complete' do - is_expected.to contain_cron('nova-manage db archive_deleted_rows').with( - :command => "nova-manage db archive_deleted_rows --max_rows #{params[:max_rows]} --until-complete >>#{params[:destination]} 2>&1", - :user => 'nova', - :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', - :user => params[:user], - :minute => params[:minute], - :hour => params[:hour], - :monthday => params[:monthday], - :month => params[:month], - :weekday => params[:weekday], - :require => 'Anchor[nova::dbsync::end]', - ) - end - end - - context 'purge is true' do - before :each do - params.merge!( - :purge => true, - ) - end - - it 'configures a cron with purge' do - is_expected.to contain_cron('nova-manage db archive_deleted_rows').with( - :command => "nova-manage db archive_deleted_rows --purge --max_rows #{params[:max_rows]} >>#{params[:destination]} 2>&1", - :user => 'nova', - :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', - :user => params[:user], - :minute => params[:minute], - :hour => params[:hour], - :monthday => params[:monthday], - :month => params[:month], - :weekday => params[:weekday], - :require => 'Anchor[nova::dbsync::end]', - ) - end - end - - context 'full purge' do - before :each do - params.merge!( - :purge => true, - :until_complete => true, - ) - end - - it 'configures a cron with all purge params' do - is_expected.to contain_cron('nova-manage db archive_deleted_rows').with( - :command => "nova-manage db archive_deleted_rows --purge --max_rows #{params[:max_rows]} --until-complete >>#{params[:destination]} 2>&1", - :user => 'nova', - :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', - :user => params[:user], - :minute => params[:minute], - :hour => params[:hour], - :monthday => params[:monthday], - :month => params[:month], - :weekday => params[:weekday], - :require => 'Anchor[nova::dbsync::end]', - ) - end - end end diff --git a/spec/classes/nova_cron_purge_shadow_tables.rb b/spec/classes/nova_cron_purge_shadow_tables.rb index c58ba0012..e332b9cfd 100644 --- a/spec/classes/nova_cron_purge_shadow_tables.rb +++ b/spec/classes/nova_cron_purge_shadow_tables.rb @@ -1,64 +1,101 @@ require 'spec_helper' +require 'date' describe 'nova::cron::purge_shadow_tables' do - let :facts do - OSDefaults.get_facts({ :osfamily => 'Debian' }) - end + shared_examples_for 'nova::cron::purge_shadow_tables' do - let :params do - { :minute => 0, - :hour => 12, - :monthday => '*', - :month => '*', - :weekday => '6', - :user => 'nova', - :destination => '/var/log/nova/nova-rowspurge.log' } - end - - context 'verbose is true' do - before :each do - params.merge!( - :verbose => true, - ) - end - - it 'configures a nova purge cron with verbose output' do - is_expected.to contain_cron('nova-manage db purge').with( - :command => "nova-manage db purge --all --verbose >>#{params[:destination]} 2>&1", + let :params do + { :minute => 0, + :hour => 12, + :monthday => '*', + :month => '*', + :weekday => '6', :user => 'nova', - :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', - :user => params[:user], - :minute => params[:minute], - :hour => params[:hour], - :monthday => params[:monthday], - :month => params[:month], - :weekday => params[:weekday], - :require => 'Anchor[nova::dbsync::end]', - ) + :destination => '/var/log/nova/nova-rowspurge.log', + :age => 10 } end + + context 'verbose is true' do + before :each do + params.merge!( + :verbose => true, + ) + end + + it 'configures a nova purge cron with verbose output' do + is_expected.to contain_cron('nova-manage db purge').with( + :command => "nova-manage db purge --before `date --date='today - #{params[:age]} days' +%D` --verbose >>#{params[:destination]} 2>&1", + :user => 'nova', + :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', + :user => params[:user], + :minute => params[:minute], + :hour => params[:hour], + :monthday => params[:monthday], + :month => params[:month], + :weekday => params[:weekday], + :require => 'Anchor[nova::dbsync::end]', + ) + end + end + + context 'verbose is false' do + before :each do + params.merge!( + :verbose => false, + ) + end + + it 'configures a nova purge cron without verbose output' do + is_expected.to contain_cron('nova-manage db purge').with( + :command => "nova-manage db purge --before `date --date='today - #{params[:age]} days' +%D` >>#{params[:destination]} 2>&1", + :user => 'nova', + :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', + :user => params[:user], + :minute => params[:minute], + :hour => params[:hour], + :monthday => params[:monthday], + :month => params[:month], + :weekday => params[:weekday], + :require => 'Anchor[nova::dbsync::end]', + ) + end + end + + context 'all_cells is true' do + before :each do + params.merge!( + :all_cells => true, + ) + end + + it 'configures a nova purge cron with all cells enabled' do + is_expected.to contain_cron('nova-manage db purge').with( + :command => "nova-manage db purge --before `date --date='today - #{params[:age]} days' +%D` --verbose --all-cells >>#{params[:destination]} 2>&1", + :user => 'nova', + :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', + :user => params[:user], + :minute => params[:minute], + :hour => params[:hour], + :monthday => params[:monthday], + :month => params[:month], + :weekday => params[:weekday], + :require => 'Anchor[nova::dbsync::end]', + ) + end + end + end - context 'verbose is false' do - before :each do - params.merge!( - :verbose => false, - ) - end + on_supported_os({ + :supported_os => OSDefaults.get_supported_os + }).each do |os,facts| + context "on #{os}" do + let (:facts) do + facts.merge!(OSDefaults.get_facts()) + end - it 'configures a nova purge cron without verbose output' do - is_expected.to contain_cron('nova-manage db purge').with( - :command => "nova-manage db purge --all >>#{params[:destination]} 2>&1", - :user => 'nova', - :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', - :user => params[:user], - :minute => params[:minute], - :hour => params[:hour], - :monthday => params[:monthday], - :month => params[:month], - :weekday => params[:weekday], - :require => 'Anchor[nova::dbsync::end]', - ) + it_configures 'nova::cron::purge_shadow_tables' end end