Add sleep in nova cron jobs
If many DB cron jobs are executed at the same time, it will trigger lock contention or DB spike. Added sleep to induce random delay before running cronjobs to avoid running all cron jobs at the same time on all hosts where this job is configured. Change-Id: I093a713a4f0ba48686de615aa8cb22b17a56917b
This commit is contained in:
parent
dc503260b0
commit
316836a08c
|
@ -61,6 +61,12 @@
|
|||
# which will automatically do a full db purge when complete.
|
||||
# Defaults to false.
|
||||
#
|
||||
# [*maxdelay*]
|
||||
# (optional) In Seconds. Should be a positive integer.
|
||||
# Induces a random delay before running the cronjob to avoid running
|
||||
# all cron jobs at the same time on all hosts this job is configured.
|
||||
# Defaults to 0.
|
||||
#
|
||||
|
||||
class nova::cron::archive_deleted_rows (
|
||||
$minute = 1,
|
||||
|
@ -73,6 +79,7 @@ class nova::cron::archive_deleted_rows (
|
|||
$destination = '/var/log/nova/nova-rowsflush.log',
|
||||
$until_complete = false,
|
||||
$purge = false,
|
||||
$maxdelay = 0,
|
||||
) {
|
||||
|
||||
include ::nova::deps
|
||||
|
@ -92,10 +99,16 @@ class nova::cron::archive_deleted_rows (
|
|||
$purge_real = ''
|
||||
}
|
||||
|
||||
if $maxdelay == 0 {
|
||||
$sleep = ''
|
||||
} else {
|
||||
$sleep = "sleep `expr \${RANDOM} \\% ${maxdelay}`; "
|
||||
}
|
||||
|
||||
$cron_cmd = 'nova-manage db archive_deleted_rows'
|
||||
|
||||
cron { 'nova-manage db archive_deleted_rows':
|
||||
command => "${cron_cmd} ${purge_real} --max_rows ${max_rows} ${until_complete_real} >>${destination} 2>&1",
|
||||
command => "${sleep}${cron_cmd} ${purge_real} --max_rows ${max_rows} ${until_complete_real} >>${destination} 2>&1",
|
||||
environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh',
|
||||
user => pick($user, $::nova::params::nova_user),
|
||||
minute => $minute,
|
||||
|
|
|
@ -55,6 +55,11 @@
|
|||
# (optional) Adds --verbose to the purge command
|
||||
# If specified, will print information about the purged rows.
|
||||
#
|
||||
# [*maxdelay*]
|
||||
# (optional) In Seconds. Should be a positive integer.
|
||||
# Induces a random delay before running the cronjob to avoid running
|
||||
# all cron jobs at the same time on all hosts this job is configured.
|
||||
# Defaults to 0.
|
||||
|
||||
class nova::cron::purge_shadow_tables (
|
||||
$minute = 0,
|
||||
|
@ -67,6 +72,7 @@ class nova::cron::purge_shadow_tables (
|
|||
$age = 14,
|
||||
$all_cells = false,
|
||||
$verbose = false,
|
||||
$maxdelay = 0,
|
||||
) {
|
||||
|
||||
include ::nova::deps
|
||||
|
@ -86,10 +92,17 @@ class nova::cron::purge_shadow_tables (
|
|||
$all_cells_real = ''
|
||||
}
|
||||
|
||||
if $maxdelay == 0 {
|
||||
$sleep = ''
|
||||
} else {
|
||||
$sleep = "sleep `expr \${RANDOM} \\% ${maxdelay}`; "
|
||||
}
|
||||
|
||||
$cron_cmd = 'nova-manage db purge'
|
||||
|
||||
cron { 'nova-manage db purge':
|
||||
command => "${cron_cmd} --before `date --date='today - ${age} days' +%D` ${verbose_real} ${all_cells_real} >>${destination} 2>&1",
|
||||
command => "${sleep}${cron_cmd} --before `date --date='today - ${age} days' +%D` ${verbose_real} \
|
||||
${all_cells_real} >>${destination} 2>&1",
|
||||
environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh',
|
||||
user => pick($user, $::nova::params::nova_user),
|
||||
minute => $minute,
|
||||
|
|
|
@ -13,6 +13,7 @@ describe 'nova::cron::archive_deleted_rows' do
|
|||
:max_rows => '100',
|
||||
:user => 'nova',
|
||||
:until_complete => false,
|
||||
:maxdelay => 0,
|
||||
:destination => '/var/log/nova/nova-rowsflush.log' }
|
||||
end
|
||||
|
||||
|
@ -103,6 +104,28 @@ describe 'nova::cron::archive_deleted_rows' do
|
|||
end
|
||||
end
|
||||
|
||||
context 'cron with maxdelay' do
|
||||
before :each do
|
||||
params.merge!(
|
||||
:maxdelay => 600
|
||||
)
|
||||
end
|
||||
|
||||
it 'configures a cron with maxdelay' do
|
||||
is_expected.to contain_cron('nova-manage db archive_deleted_rows').with(
|
||||
:command => "sleep `expr ${RANDOM} \\% #{params[:maxdelay]}`; nova-manage db archive_deleted_rows --max_rows #{params[:max_rows]} >>#{params[:destination]} 2>&1",
|
||||
:environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh',
|
||||
:user => params[:user],
|
||||
:minute => params[:minute],
|
||||
:hour => params[:hour],
|
||||
:monthday => params[:monthday],
|
||||
:month => params[:month],
|
||||
:weekday => params[:weekday],
|
||||
:require => 'Anchor[nova::dbsync::end]',
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
on_supported_os({
|
||||
|
|
|
@ -12,6 +12,7 @@ describe 'nova::cron::purge_shadow_tables' do
|
|||
:month => '*',
|
||||
:weekday => '6',
|
||||
:user => 'nova',
|
||||
:maxdelay => 0,
|
||||
:destination => '/var/log/nova/nova-rowspurge.log',
|
||||
:age => 10 }
|
||||
end
|
||||
|
@ -85,6 +86,28 @@ describe 'nova::cron::purge_shadow_tables' do
|
|||
end
|
||||
end
|
||||
|
||||
context 'cron with maxdelay' do
|
||||
before :each do
|
||||
params.merge!(
|
||||
:maxdelay => 600
|
||||
)
|
||||
end
|
||||
|
||||
it 'configures a nova purge cron with maxdelay' do
|
||||
is_expected.to contain_cron('nova-manage db purge').with(
|
||||
:command => "sleep `expr ${RANDOM} \\% #{params[:maxdelay]}`; nova-manage db purge --before `date --date='today - #{params[:age]} days' +%D` --verbose --all-cells >>#{params[:destination]} 2>&1",
|
||||
:environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh',
|
||||
:user => params[:user],
|
||||
:minute => params[:minute],
|
||||
:hour => params[:hour],
|
||||
:monthday => params[:monthday],
|
||||
:month => params[:month],
|
||||
:weekday => params[:weekday],
|
||||
:require => 'Anchor[nova::dbsync::end]',
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
on_supported_os({
|
||||
|
|
Loading…
Reference in New Issue