From 1f49e5b58e4ecfbc795bd18f2553948f6b578d2a Mon Sep 17 00:00:00 2001
From: Swann Croiset <scroiset@mirantis.com>
Date: Tue, 5 Apr 2016 09:53:24 +0200
Subject: [PATCH] Increase the Heka poolsize on controllers

On controller nodes, the Heka poolsize must be increased to handle the load
generated by derived metrics from logs otherwise a deadlock
can happen in the filter plugins and block heka.

Fixes-bug: #1557388

Change-Id: I74362011d32d413f244c6cdb6e4625ed96759df0
(cherry picked from commit 96df47af7385578280a68863b55416da7dd8bfdb)
---
 deployment_scripts/puppet/manifests/base.pp     | 17 ++++++++++++++---
 .../puppet/modules/heka/manifests/init.pp       |  8 ++++++++
 .../modules/heka/templates/global.toml.erb      |  3 +++
 .../puppet/modules/lma_collector/README.md      |  1 +
 .../modules/lma_collector/manifests/init.pp     |  3 +++
 5 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/deployment_scripts/puppet/manifests/base.pp b/deployment_scripts/puppet/manifests/base.pp
index b23efa4b0..c484b06fa 100644
--- a/deployment_scripts/puppet/manifests/base.pp
+++ b/deployment_scripts/puppet/manifests/base.pp
@@ -87,10 +87,21 @@ case $::osfamily {
   }
 }
 
+if $is_controller {
+  # On controller nodes the log parsing can generate a lot of http_metrics
+  # which can block heka (idle packs). It was observed that a poolsize set to 200
+  # solves the issue.
+  $poolsize = 200
+} else {
+  # For other nodes, the poolsize is set to 100 (the Heka default value)
+  $poolsize = 100
+}
+
 class { 'lma_collector':
-  tags   => $tags,
-  user   => $heka_user,
-  groups => $additional_groups,
+  tags     => $tags,
+  user     => $heka_user,
+  groups   => $additional_groups,
+  poolsize => $poolsize,
 }
 
 # On controller nodes the LMA collector service is managed by Pacemaker, so we
diff --git a/deployment_scripts/puppet/modules/heka/manifests/init.pp b/deployment_scripts/puppet/modules/heka/manifests/init.pp
index 6183daddd..ef16d4c9d 100644
--- a/deployment_scripts/puppet/modules/heka/manifests/init.pp
+++ b/deployment_scripts/puppet/modules/heka/manifests/init.pp
@@ -51,6 +51,9 @@
 # [*dashboard_port*]
 #   The listening port for the Heka dashboard (default: 4352).
 #
+# [*poolsize*]
+#   The pool size of maximum messages that can exist (default: 100).
+#
 # [*internal_statistics*]
 #   Whether or not to dump Heka internal statistics to stdout at a regular
 #   interval (currently every hour).
@@ -82,10 +85,15 @@ class heka (
   $max_timer_inject = $heka::params::max_timer_inject,
   $dashboard_address = $heka::params::dashboard_address,
   $dashboard_port = $heka::params::dashboard_port,
+  $poolsize = undef,
   $pre_script = undef,
   $internal_statistics = $heka::params::internal_statistics,
 ) inherits heka::params {
 
+  if $poolsize {
+    validate_integer($poolsize)
+  }
+
   $hekad_wrapper = "/usr/local/bin/${service_name}_wrapper"
   $base_dir      = "/var/cache/${service_name}"
   $log_file      = "/var/log/${service_name}.log"
diff --git a/deployment_scripts/puppet/modules/heka/templates/global.toml.erb b/deployment_scripts/puppet/modules/heka/templates/global.toml.erb
index 2ad5cf40e..b09671bc4 100644
--- a/deployment_scripts/puppet/modules/heka/templates/global.toml.erb
+++ b/deployment_scripts/puppet/modules/heka/templates/global.toml.erb
@@ -13,3 +13,6 @@ max_process_inject = <%= @max_process_inject %>
 <% if @max_timer_inject -%>
 max_timer_inject = <%= @max_timer_inject %>
 <% end -%>
+<% if @poolsize -%>
+poolsize = <%= @poolsize %>
+<% end -%>
diff --git a/deployment_scripts/puppet/modules/lma_collector/README.md b/deployment_scripts/puppet/modules/lma_collector/README.md
index b662a0178..9f6afca75 100644
--- a/deployment_scripts/puppet/modules/lma_collector/README.md
+++ b/deployment_scripts/puppet/modules/lma_collector/README.md
@@ -471,6 +471,7 @@ Main class. Install and configure the main components of the LMA collector.
 * `tags`: *Optional*. Fields added to Heka messages. Valid options: a hash. Default: `{}`.
 * `user`: *Optional*. User the Heka service is run as. You may have to use `'root'` on some systems for the Heka service to be able to access log files, run additional commands, ... Valid options: a string.  Default: `'heka'`.
 * `groups`: *Optional*. Additional groups to add to the user running the Heka service. Ignored if the Heka service is run as "root". Valid options: an array of strings. Default: `['syslog', 'adm']`.
+* `poolsize`: *Optional*. The pool size of maximum messages that can exist (default: 100).
 
 #### Class: `lma_collector::elasticsearch`
 
diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp
index 26324bd01..5b8272ee9 100644
--- a/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp
+++ b/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp
@@ -33,10 +33,12 @@ class lma_collector (
   $tags = $lma_collector::params::tags,
   $user = undef,
   $groups = [],
+  $poolsize = 100,
 ) inherits lma_collector::params {
   include heka::params
 
   validate_hash($tags)
+  validate_integer($poolsize)
 
   $service_name = $lma_collector::params::service_name
   $config_dir = $lma_collector::params::config_dir
@@ -58,6 +60,7 @@ class lma_collector (
     max_message_size    => $lma_collector::params::hekad_max_message_size,
     max_process_inject  => $lma_collector::params::hekad_max_process_inject,
     max_timer_inject    => $lma_collector::params::hekad_max_timer_inject,
+    poolsize            => $poolsize,
   }
 
   # Copy our Lua modules to Heka's modules directory so they're available for