opendev/system-config
Code Issues Proposed changes

Run fewer es queries with elastic_recheck.

Browse Source 
Currently elastic-recheck is hammering the ElasticSearch cluster every
15 minutes, basically bringing everything else to a halt while these
queries are processed. Spread elastic-recheck periodic queries out, run
each once an hour and 20 minutes after the previous set of queries.

Also, remove the auto triggering when files change, instead we wait for
the hour to roll over and give us new data.

Change-Id: Ibf5dad1071f65f8d676e8b9a0ec4331bada9943f
This commit is contained in:
Clark Boylan 2014-02-10 16:25:24 -08:00
parent 60fd91d2c0
commit e24b23bc51
2 changed files with 4 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
modules/elastic_recheck/manifests/cron.pp
View File

@ -25,7 +25,7 @@ class elastic_recheck::cron () {
cron { 'elastic-recheck-all': cron { 'elastic-recheck-all':
user => 'recheck', user => 'recheck',
minute => '*/15', minute => '0',
hour => '*', hour => '*',
command => "cd ${er_state_path} && er_safe_run.sh ${graph_all_cmd}", command => "cd ${er_state_path} && er_safe_run.sh ${graph_all_cmd}",
environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin', environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin',
@ -34,9 +34,9 @@ class elastic_recheck::cron () {
cron { 'elastic-recheck-gate': cron { 'elastic-recheck-gate':
user => 'recheck', user => 'recheck',
minute => '*/15', minute => '20',
hour => '*', hour => '*',
command => "sleep $((RANDOM\%60+90)) && cd ${er_state_path} && er_safe_run.sh ${graph_gate_cmd}", command => "cd ${er_state_path} && er_safe_run.sh ${graph_gate_cmd}",
environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin', environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin',
require => Class['elastic_recheck'] require => Class['elastic_recheck']
} }
@ -44,7 +44,7 @@ class elastic_recheck::cron () {
cron { 'elastic-recheck-uncat': cron { 'elastic-recheck-uncat':
user => 'recheck', user => 'recheck',
minute => '59', minute => '40',
hour => '*', hour => '*',
command => "cd ${er_state_path} && er_safe_run.sh ${uncat_cmd}", command => "cd ${er_state_path} && er_safe_run.sh ${uncat_cmd}",
environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin', environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin',

30
modules/elastic_recheck/manifests/init.pp
View File

@ -46,36 +46,6 @@ class elastic_recheck (
source => 'https://git.openstack.org/openstack-infra/elastic-recheck', source => 'https://git.openstack.org/openstack-infra/elastic-recheck',
} }
exec { 'run_er_graph_all':
command => "er_safe_run.sh ${graph_all_cmd}",
path => '/usr/local/bin:/usr/bin:/bin/',
cwd => $recheck_state_dir,
user => 'recheck',
refreshonly => true,
require => File['/usr/local/bin/er_safe_run.sh'],
subscribe => Vcsrepo['/opt/elastic-recheck'],
}
exec { 'run_er_graph_gate_only':
command => "er_safe_run.sh ${graph_gate_cmd}",
path => '/usr/local/bin:/usr/bin:/bin/',
cwd => $recheck_state_dir,
user => 'recheck',
refreshonly => true,
require => File['/usr/local/bin/er_safe_run.sh'],
subscribe => Vcsrepo['/opt/elastic-recheck'],
}
exec { 'run_er_uncat':
command => "er_safe_run.sh ${uncat_cmd}",
path => '/usr/local/bin:/usr/bin:/bin/',
cwd => $recheck_state_dir,
user => 'recheck',
refreshonly => true,
require => File['/usr/local/bin/er_safe_run.sh'],
subscribe => Vcsrepo['/opt/elastic-recheck'],
}
include pip include pip
exec { 'install_elastic-recheck' : exec { 'install_elastic-recheck' :
command => 'pip install /opt/elastic-recheck', command => 'pip install /opt/elastic-recheck',