Add sleep in nova cron jobs

If many DB cron jobs are executed at the same time, it will
trigger lock contention or DB spike.

Added sleep to induce random delay before running cronjobs to avoid
running all cron jobs at the same time on all hosts where this
job is configured.

Change-Id: I093a713a4f0ba48686de615aa8cb22b17a56917b
This commit is contained in:
Rajesh Tailor 2018-07-20 17:25:54 +05:30
parent dc503260b0
commit 316836a08c
4 changed files with 74 additions and 2 deletions

View File

@ -61,6 +61,12 @@
# which will automatically do a full db purge when complete.
# Defaults to false.
#
# [*maxdelay*]
# (optional) In Seconds. Should be a positive integer.
# Induces a random delay before running the cronjob to avoid running
# all cron jobs at the same time on all hosts this job is configured.
# Defaults to 0.
#
class nova::cron::archive_deleted_rows (
$minute = 1,
@ -73,6 +79,7 @@ class nova::cron::archive_deleted_rows (
$destination = '/var/log/nova/nova-rowsflush.log',
$until_complete = false,
$purge = false,
$maxdelay = 0,
) {
include ::nova::deps
@ -92,10 +99,16 @@ class nova::cron::archive_deleted_rows (
$purge_real = ''
}
if $maxdelay == 0 {
$sleep = ''
} else {
$sleep = "sleep `expr \${RANDOM} \\% ${maxdelay}`; "
}
$cron_cmd = 'nova-manage db archive_deleted_rows'
cron { 'nova-manage db archive_deleted_rows':
command => "${cron_cmd} ${purge_real} --max_rows ${max_rows} ${until_complete_real} >>${destination} 2>&1",
command => "${sleep}${cron_cmd} ${purge_real} --max_rows ${max_rows} ${until_complete_real} >>${destination} 2>&1",
environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh',
user => pick($user, $::nova::params::nova_user),
minute => $minute,

View File

@ -55,6 +55,11 @@
# (optional) Adds --verbose to the purge command
# If specified, will print information about the purged rows.
#
# [*maxdelay*]
# (optional) In Seconds. Should be a positive integer.
# Induces a random delay before running the cronjob to avoid running
# all cron jobs at the same time on all hosts this job is configured.
# Defaults to 0.
class nova::cron::purge_shadow_tables (
$minute = 0,
@ -67,6 +72,7 @@ class nova::cron::purge_shadow_tables (
$age = 14,
$all_cells = false,
$verbose = false,
$maxdelay = 0,
) {
include ::nova::deps
@ -86,10 +92,17 @@ class nova::cron::purge_shadow_tables (
$all_cells_real = ''
}
if $maxdelay == 0 {
$sleep = ''
} else {
$sleep = "sleep `expr \${RANDOM} \\% ${maxdelay}`; "
}
$cron_cmd = 'nova-manage db purge'
cron { 'nova-manage db purge':
command => "${cron_cmd} --before `date --date='today - ${age} days' +%D` ${verbose_real} ${all_cells_real} >>${destination} 2>&1",
command => "${sleep}${cron_cmd} --before `date --date='today - ${age} days' +%D` ${verbose_real} \
${all_cells_real} >>${destination} 2>&1",
environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh',
user => pick($user, $::nova::params::nova_user),
minute => $minute,

View File

@ -13,6 +13,7 @@ describe 'nova::cron::archive_deleted_rows' do
:max_rows => '100',
:user => 'nova',
:until_complete => false,
:maxdelay => 0,
:destination => '/var/log/nova/nova-rowsflush.log' }
end
@ -103,6 +104,28 @@ describe 'nova::cron::archive_deleted_rows' do
end
end
context 'cron with maxdelay' do
before :each do
params.merge!(
:maxdelay => 600
)
end
it 'configures a cron with maxdelay' do
is_expected.to contain_cron('nova-manage db archive_deleted_rows').with(
:command => "sleep `expr ${RANDOM} \\% #{params[:maxdelay]}`; nova-manage db archive_deleted_rows --max_rows #{params[:max_rows]} >>#{params[:destination]} 2>&1",
:environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh',
:user => params[:user],
:minute => params[:minute],
:hour => params[:hour],
:monthday => params[:monthday],
:month => params[:month],
:weekday => params[:weekday],
:require => 'Anchor[nova::dbsync::end]',
)
end
end
end
on_supported_os({

View File

@ -12,6 +12,7 @@ describe 'nova::cron::purge_shadow_tables' do
:month => '*',
:weekday => '6',
:user => 'nova',
:maxdelay => 0,
:destination => '/var/log/nova/nova-rowspurge.log',
:age => 10 }
end
@ -85,6 +86,28 @@ describe 'nova::cron::purge_shadow_tables' do
end
end
context 'cron with maxdelay' do
before :each do
params.merge!(
:maxdelay => 600
)
end
it 'configures a nova purge cron with maxdelay' do
is_expected.to contain_cron('nova-manage db purge').with(
:command => "sleep `expr ${RANDOM} \\% #{params[:maxdelay]}`; nova-manage db purge --before `date --date='today - #{params[:age]} days' +%D` --verbose --all-cells >>#{params[:destination]} 2>&1",
:environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh',
:user => params[:user],
:minute => params[:minute],
:hour => params[:hour],
:monthday => params[:monthday],
:month => params[:month],
:weekday => params[:weekday],
:require => 'Anchor[nova::dbsync::end]',
)
end
end
end
on_supported_os({