Increase the Heka poolsize on controllers
On controller nodes, the Heka poolsize must be increased to handle the load
generated by derived metrics from logs otherwise a deadlock
can happen in the filter plugins and block heka.
Fixes-bug: #1557388
Change-Id: I74362011d32d413f244c6cdb6e4625ed96759df0
(cherry picked from commit 96df47af73
)
This commit is contained in:
parent
c2888c1299
commit
1f49e5b58e
|
@ -87,10 +87,21 @@ case $::osfamily {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if $is_controller {
|
||||||
|
# On controller nodes the log parsing can generate a lot of http_metrics
|
||||||
|
# which can block heka (idle packs). It was observed that a poolsize set to 200
|
||||||
|
# solves the issue.
|
||||||
|
$poolsize = 200
|
||||||
|
} else {
|
||||||
|
# For other nodes, the poolsize is set to 100 (the Heka default value)
|
||||||
|
$poolsize = 100
|
||||||
|
}
|
||||||
|
|
||||||
class { 'lma_collector':
|
class { 'lma_collector':
|
||||||
tags => $tags,
|
tags => $tags,
|
||||||
user => $heka_user,
|
user => $heka_user,
|
||||||
groups => $additional_groups,
|
groups => $additional_groups,
|
||||||
|
poolsize => $poolsize,
|
||||||
}
|
}
|
||||||
|
|
||||||
# On controller nodes the LMA collector service is managed by Pacemaker, so we
|
# On controller nodes the LMA collector service is managed by Pacemaker, so we
|
||||||
|
|
|
@ -51,6 +51,9 @@
|
||||||
# [*dashboard_port*]
|
# [*dashboard_port*]
|
||||||
# The listening port for the Heka dashboard (default: 4352).
|
# The listening port for the Heka dashboard (default: 4352).
|
||||||
#
|
#
|
||||||
|
# [*poolsize*]
|
||||||
|
# The pool size of maximum messages that can exist (default: 100).
|
||||||
|
#
|
||||||
# [*internal_statistics*]
|
# [*internal_statistics*]
|
||||||
# Whether or not to dump Heka internal statistics to stdout at a regular
|
# Whether or not to dump Heka internal statistics to stdout at a regular
|
||||||
# interval (currently every hour).
|
# interval (currently every hour).
|
||||||
|
@ -82,10 +85,15 @@ class heka (
|
||||||
$max_timer_inject = $heka::params::max_timer_inject,
|
$max_timer_inject = $heka::params::max_timer_inject,
|
||||||
$dashboard_address = $heka::params::dashboard_address,
|
$dashboard_address = $heka::params::dashboard_address,
|
||||||
$dashboard_port = $heka::params::dashboard_port,
|
$dashboard_port = $heka::params::dashboard_port,
|
||||||
|
$poolsize = undef,
|
||||||
$pre_script = undef,
|
$pre_script = undef,
|
||||||
$internal_statistics = $heka::params::internal_statistics,
|
$internal_statistics = $heka::params::internal_statistics,
|
||||||
) inherits heka::params {
|
) inherits heka::params {
|
||||||
|
|
||||||
|
if $poolsize {
|
||||||
|
validate_integer($poolsize)
|
||||||
|
}
|
||||||
|
|
||||||
$hekad_wrapper = "/usr/local/bin/${service_name}_wrapper"
|
$hekad_wrapper = "/usr/local/bin/${service_name}_wrapper"
|
||||||
$base_dir = "/var/cache/${service_name}"
|
$base_dir = "/var/cache/${service_name}"
|
||||||
$log_file = "/var/log/${service_name}.log"
|
$log_file = "/var/log/${service_name}.log"
|
||||||
|
|
|
@ -13,3 +13,6 @@ max_process_inject = <%= @max_process_inject %>
|
||||||
<% if @max_timer_inject -%>
|
<% if @max_timer_inject -%>
|
||||||
max_timer_inject = <%= @max_timer_inject %>
|
max_timer_inject = <%= @max_timer_inject %>
|
||||||
<% end -%>
|
<% end -%>
|
||||||
|
<% if @poolsize -%>
|
||||||
|
poolsize = <%= @poolsize %>
|
||||||
|
<% end -%>
|
||||||
|
|
|
@ -471,6 +471,7 @@ Main class. Install and configure the main components of the LMA collector.
|
||||||
* `tags`: *Optional*. Fields added to Heka messages. Valid options: a hash. Default: `{}`.
|
* `tags`: *Optional*. Fields added to Heka messages. Valid options: a hash. Default: `{}`.
|
||||||
* `user`: *Optional*. User the Heka service is run as. You may have to use `'root'` on some systems for the Heka service to be able to access log files, run additional commands, ... Valid options: a string. Default: `'heka'`.
|
* `user`: *Optional*. User the Heka service is run as. You may have to use `'root'` on some systems for the Heka service to be able to access log files, run additional commands, ... Valid options: a string. Default: `'heka'`.
|
||||||
* `groups`: *Optional*. Additional groups to add to the user running the Heka service. Ignored if the Heka service is run as "root". Valid options: an array of strings. Default: `['syslog', 'adm']`.
|
* `groups`: *Optional*. Additional groups to add to the user running the Heka service. Ignored if the Heka service is run as "root". Valid options: an array of strings. Default: `['syslog', 'adm']`.
|
||||||
|
* `poolsize`: *Optional*. The pool size of maximum messages that can exist (default: 100).
|
||||||
|
|
||||||
#### Class: `lma_collector::elasticsearch`
|
#### Class: `lma_collector::elasticsearch`
|
||||||
|
|
||||||
|
|
|
@ -33,10 +33,12 @@ class lma_collector (
|
||||||
$tags = $lma_collector::params::tags,
|
$tags = $lma_collector::params::tags,
|
||||||
$user = undef,
|
$user = undef,
|
||||||
$groups = [],
|
$groups = [],
|
||||||
|
$poolsize = 100,
|
||||||
) inherits lma_collector::params {
|
) inherits lma_collector::params {
|
||||||
include heka::params
|
include heka::params
|
||||||
|
|
||||||
validate_hash($tags)
|
validate_hash($tags)
|
||||||
|
validate_integer($poolsize)
|
||||||
|
|
||||||
$service_name = $lma_collector::params::service_name
|
$service_name = $lma_collector::params::service_name
|
||||||
$config_dir = $lma_collector::params::config_dir
|
$config_dir = $lma_collector::params::config_dir
|
||||||
|
@ -58,6 +60,7 @@ class lma_collector (
|
||||||
max_message_size => $lma_collector::params::hekad_max_message_size,
|
max_message_size => $lma_collector::params::hekad_max_message_size,
|
||||||
max_process_inject => $lma_collector::params::hekad_max_process_inject,
|
max_process_inject => $lma_collector::params::hekad_max_process_inject,
|
||||||
max_timer_inject => $lma_collector::params::hekad_max_timer_inject,
|
max_timer_inject => $lma_collector::params::hekad_max_timer_inject,
|
||||||
|
poolsize => $poolsize,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Copy our Lua modules to Heka's modules directory so they're available for
|
# Copy our Lua modules to Heka's modules directory so they're available for
|
||||||
|
|
Loading…
Reference in New Issue