Increase the Heka poolsize on controllers

On controller nodes, the Heka poolsize must be increased to handle the load
generated by derived metrics from logs otherwise a deadlock
can happen in the filter plugins and block heka.

Fixes-bug: #1557388

Change-Id: I74362011d32d413f244c6cdb6e4625ed96759df0
(cherry picked from commit 96df47af73)
This commit is contained in:
Swann Croiset 2016-04-05 09:53:24 +02:00
parent c2888c1299
commit 1f49e5b58e
5 changed files with 29 additions and 3 deletions

View File

@ -87,10 +87,21 @@ case $::osfamily {
}
}
if $is_controller {
# On controller nodes the log parsing can generate a lot of http_metrics
# which can block heka (idle packs). It was observed that a poolsize set to 200
# solves the issue.
$poolsize = 200
} else {
# For other nodes, the poolsize is set to 100 (the Heka default value)
$poolsize = 100
}
class { 'lma_collector':
tags => $tags,
user => $heka_user,
groups => $additional_groups,
tags => $tags,
user => $heka_user,
groups => $additional_groups,
poolsize => $poolsize,
}
# On controller nodes the LMA collector service is managed by Pacemaker, so we

View File

@ -51,6 +51,9 @@
# [*dashboard_port*]
# The listening port for the Heka dashboard (default: 4352).
#
# [*poolsize*]
# The pool size of maximum messages that can exist (default: 100).
#
# [*internal_statistics*]
# Whether or not to dump Heka internal statistics to stdout at a regular
# interval (currently every hour).
@ -82,10 +85,15 @@ class heka (
$max_timer_inject = $heka::params::max_timer_inject,
$dashboard_address = $heka::params::dashboard_address,
$dashboard_port = $heka::params::dashboard_port,
$poolsize = undef,
$pre_script = undef,
$internal_statistics = $heka::params::internal_statistics,
) inherits heka::params {
if $poolsize {
validate_integer($poolsize)
}
$hekad_wrapper = "/usr/local/bin/${service_name}_wrapper"
$base_dir = "/var/cache/${service_name}"
$log_file = "/var/log/${service_name}.log"

View File

@ -13,3 +13,6 @@ max_process_inject = <%= @max_process_inject %>
<% if @max_timer_inject -%>
max_timer_inject = <%= @max_timer_inject %>
<% end -%>
<% if @poolsize -%>
poolsize = <%= @poolsize %>
<% end -%>

View File

@ -471,6 +471,7 @@ Main class. Install and configure the main components of the LMA collector.
* `tags`: *Optional*. Fields added to Heka messages. Valid options: a hash. Default: `{}`.
* `user`: *Optional*. User the Heka service is run as. You may have to use `'root'` on some systems for the Heka service to be able to access log files, run additional commands, ... Valid options: a string. Default: `'heka'`.
* `groups`: *Optional*. Additional groups to add to the user running the Heka service. Ignored if the Heka service is run as "root". Valid options: an array of strings. Default: `['syslog', 'adm']`.
* `poolsize`: *Optional*. The pool size of maximum messages that can exist (default: 100).
#### Class: `lma_collector::elasticsearch`

View File

@ -33,10 +33,12 @@ class lma_collector (
$tags = $lma_collector::params::tags,
$user = undef,
$groups = [],
$poolsize = 100,
) inherits lma_collector::params {
include heka::params
validate_hash($tags)
validate_integer($poolsize)
$service_name = $lma_collector::params::service_name
$config_dir = $lma_collector::params::config_dir
@ -58,6 +60,7 @@ class lma_collector (
max_message_size => $lma_collector::params::hekad_max_message_size,
max_process_inject => $lma_collector::params::hekad_max_process_inject,
max_timer_inject => $lma_collector::params::hekad_max_timer_inject,
poolsize => $poolsize,
}
# Copy our Lua modules to Heka's modules directory so they're available for