From 316836a08cb76a8995121e9f1e1859295e15c820 Mon Sep 17 00:00:00 2001 From: Rajesh Tailor Date: Fri, 20 Jul 2018 17:25:54 +0530 Subject: [PATCH] Add sleep in nova cron jobs If many DB cron jobs are executed at the same time, it will trigger lock contention or DB spike. Added sleep to induce random delay before running cronjobs to avoid running all cron jobs at the same time on all hosts where this job is configured. Change-Id: I093a713a4f0ba48686de615aa8cb22b17a56917b --- manifests/cron/archive_deleted_rows.pp | 15 +++++++++++- manifests/cron/purge_shadow_tables.pp | 15 +++++++++++- .../nova_cron_archive_deleted_rows_spec.rb | 23 +++++++++++++++++++ spec/classes/nova_cron_purge_shadow_tables.rb | 23 +++++++++++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/manifests/cron/archive_deleted_rows.pp b/manifests/cron/archive_deleted_rows.pp index 5bed5d4db..7ecb2853e 100644 --- a/manifests/cron/archive_deleted_rows.pp +++ b/manifests/cron/archive_deleted_rows.pp @@ -61,6 +61,12 @@ # which will automatically do a full db purge when complete. # Defaults to false. # +# [*maxdelay*] +# (optional) In Seconds. Should be a positive integer. +# Induces a random delay before running the cronjob to avoid running +# all cron jobs at the same time on all hosts this job is configured. +# Defaults to 0. +# class nova::cron::archive_deleted_rows ( $minute = 1, @@ -73,6 +79,7 @@ class nova::cron::archive_deleted_rows ( $destination = '/var/log/nova/nova-rowsflush.log', $until_complete = false, $purge = false, + $maxdelay = 0, ) { include ::nova::deps @@ -92,10 +99,16 @@ class nova::cron::archive_deleted_rows ( $purge_real = '' } + if $maxdelay == 0 { + $sleep = '' + } else { + $sleep = "sleep `expr \${RANDOM} \\% ${maxdelay}`; " + } + $cron_cmd = 'nova-manage db archive_deleted_rows' cron { 'nova-manage db archive_deleted_rows': - command => "${cron_cmd} ${purge_real} --max_rows ${max_rows} ${until_complete_real} >>${destination} 2>&1", + command => "${sleep}${cron_cmd} ${purge_real} --max_rows ${max_rows} ${until_complete_real} >>${destination} 2>&1", environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', user => pick($user, $::nova::params::nova_user), minute => $minute, diff --git a/manifests/cron/purge_shadow_tables.pp b/manifests/cron/purge_shadow_tables.pp index 6ed8db528..440358f19 100644 --- a/manifests/cron/purge_shadow_tables.pp +++ b/manifests/cron/purge_shadow_tables.pp @@ -55,6 +55,11 @@ # (optional) Adds --verbose to the purge command # If specified, will print information about the purged rows. # +# [*maxdelay*] +# (optional) In Seconds. Should be a positive integer. +# Induces a random delay before running the cronjob to avoid running +# all cron jobs at the same time on all hosts this job is configured. +# Defaults to 0. class nova::cron::purge_shadow_tables ( $minute = 0, @@ -67,6 +72,7 @@ class nova::cron::purge_shadow_tables ( $age = 14, $all_cells = false, $verbose = false, + $maxdelay = 0, ) { include ::nova::deps @@ -86,10 +92,17 @@ class nova::cron::purge_shadow_tables ( $all_cells_real = '' } + if $maxdelay == 0 { + $sleep = '' + } else { + $sleep = "sleep `expr \${RANDOM} \\% ${maxdelay}`; " + } + $cron_cmd = 'nova-manage db purge' cron { 'nova-manage db purge': - command => "${cron_cmd} --before `date --date='today - ${age} days' +%D` ${verbose_real} ${all_cells_real} >>${destination} 2>&1", + command => "${sleep}${cron_cmd} --before `date --date='today - ${age} days' +%D` ${verbose_real} \ + ${all_cells_real} >>${destination} 2>&1", environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', user => pick($user, $::nova::params::nova_user), minute => $minute, diff --git a/spec/classes/nova_cron_archive_deleted_rows_spec.rb b/spec/classes/nova_cron_archive_deleted_rows_spec.rb index 558d42a46..5f996bb94 100644 --- a/spec/classes/nova_cron_archive_deleted_rows_spec.rb +++ b/spec/classes/nova_cron_archive_deleted_rows_spec.rb @@ -13,6 +13,7 @@ describe 'nova::cron::archive_deleted_rows' do :max_rows => '100', :user => 'nova', :until_complete => false, + :maxdelay => 0, :destination => '/var/log/nova/nova-rowsflush.log' } end @@ -103,6 +104,28 @@ describe 'nova::cron::archive_deleted_rows' do end end + context 'cron with maxdelay' do + before :each do + params.merge!( + :maxdelay => 600 + ) + end + + it 'configures a cron with maxdelay' do + is_expected.to contain_cron('nova-manage db archive_deleted_rows').with( + :command => "sleep `expr ${RANDOM} \\% #{params[:maxdelay]}`; nova-manage db archive_deleted_rows --max_rows #{params[:max_rows]} >>#{params[:destination]} 2>&1", + :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', + :user => params[:user], + :minute => params[:minute], + :hour => params[:hour], + :monthday => params[:monthday], + :month => params[:month], + :weekday => params[:weekday], + :require => 'Anchor[nova::dbsync::end]', + ) + end + end + end on_supported_os({ diff --git a/spec/classes/nova_cron_purge_shadow_tables.rb b/spec/classes/nova_cron_purge_shadow_tables.rb index e332b9cfd..3b4759b72 100644 --- a/spec/classes/nova_cron_purge_shadow_tables.rb +++ b/spec/classes/nova_cron_purge_shadow_tables.rb @@ -12,6 +12,7 @@ describe 'nova::cron::purge_shadow_tables' do :month => '*', :weekday => '6', :user => 'nova', + :maxdelay => 0, :destination => '/var/log/nova/nova-rowspurge.log', :age => 10 } end @@ -85,6 +86,28 @@ describe 'nova::cron::purge_shadow_tables' do end end + context 'cron with maxdelay' do + before :each do + params.merge!( + :maxdelay => 600 + ) + end + + it 'configures a nova purge cron with maxdelay' do + is_expected.to contain_cron('nova-manage db purge').with( + :command => "sleep `expr ${RANDOM} \\% #{params[:maxdelay]}`; nova-manage db purge --before `date --date='today - #{params[:age]} days' +%D` --verbose --all-cells >>#{params[:destination]} 2>&1", + :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh', + :user => params[:user], + :minute => params[:minute], + :hour => params[:hour], + :monthday => params[:monthday], + :month => params[:month], + :weekday => params[:weekday], + :require => 'Anchor[nova::dbsync::end]', + ) + end + end + end on_supported_os({