Browse Source

Increase the Heka poolsize on controllers

On controller nodes, the Heka poolsize must be increased to handle the load
generated by derived metrics from logs otherwise a deadlock
can happen in the filter plugins and block heka.

Fixes-bug: #1557388
(cherry picked from commit 96df47af73)

Conflicts:
	deployment_scripts/puppet/manifests/base.pp
	deployment_scripts/puppet/modules/lma_collector/manifests/init.pp

Change-Id: I74362011d32d413f244c6cdb6e4625ed96759df0
tags/0.8.1rc3^0
Swann Croiset 3 years ago
parent
commit
346425f04d

+ 8
- 0
deployment_scripts/puppet/manifests/base.pp View File

@@ -83,6 +83,11 @@ if $is_controller{
83 83
   $rabbitmq_password = $rabbit['password']
84 84
   $wait_delay        = 30
85 85
 
86
+  # On controller nodes the log parsing can generate a lot of http_metrics
87
+  # which can block heka (idle packs). It was observed that a poolsize set to 200
88
+  # solves the issue.
89
+  $poolsize = 200
90
+
86 91
   #  file { $pre_script:
87 92
   #    ensure  => present,
88 93
   #    owner   => 'root',
@@ -93,6 +98,8 @@ if $is_controller{
93 98
   #  }
94 99
 } else {
95 100
   $pre_script = undef
101
+  # For other nodes, the poolsize is set to 100 (the Heka default value)
102
+  $poolsize = 100
96 103
 }
97 104
 
98 105
 class { 'lma_collector':
@@ -100,6 +107,7 @@ class { 'lma_collector':
100 107
   groups            => $additional_groups,
101 108
   pacemaker_managed => $pacemaker_managed,
102 109
   rabbitmq_resource => $rabbitmq_resource,
110
+  poolsize          => $poolsize,
103 111
 }
104 112
 
105 113
 if $elasticsearch_mode != 'disabled' {

+ 8
- 0
deployment_scripts/puppet/modules/heka/manifests/init.pp View File

@@ -53,6 +53,9 @@
53 53
 # [*dashboard_port*]
54 54
 #   The listening port for the Heka dashboard (default: 4352).
55 55
 #
56
+# [*poolsize*]
57
+#   The pool size of maximum messages that can exist (default: 100).
58
+#
56 59
 # [*internal_statistics*]
57 60
 #   Whether or not to dump Heka internal statistics to stdout at a regular
58 61
 #   interval (currently every hour).
@@ -84,10 +87,15 @@ class heka (
84 87
   $max_timer_inject = $heka::params::max_timer_inject,
85 88
   $dashboard_address = $heka::params::dashboard_address,
86 89
   $dashboard_port = $heka::params::dashboard_port,
90
+  $poolsize = undef,
87 91
   $pre_script = undef,
88 92
   $internal_statistics = $heka::params::internal_statistics,
89 93
 ) inherits heka::params {
90 94
 
95
+  if $poolsize {
96
+    validate_integer($poolsize)
97
+  }
98
+
91 99
   $hekad_wrapper = "/usr/local/bin/${service_name}_wrapper"
92 100
   $base_dir      = "/var/cache/${service_name}"
93 101
   $log_file      = "/var/log/${service_name}.log"

+ 3
- 0
deployment_scripts/puppet/modules/heka/templates/global.toml.erb View File

@@ -13,3 +13,6 @@ max_process_inject = <%= @max_process_inject %>
13 13
 <% if @max_timer_inject -%>
14 14
 max_timer_inject = <%= @max_timer_inject %>
15 15
 <% end -%>
16
+<% if @poolsize -%>
17
+poolsize = <%= @poolsize %>
18
+<% end -%>

+ 3
- 0
deployment_scripts/puppet/modules/lma_collector/manifests/init.pp View File

@@ -33,6 +33,7 @@ class lma_collector (
33 33
   $tags = $lma_collector::params::tags,
34 34
   $groups = [],
35 35
   $pre_script = undef,
36
+  $poolsize = 100,
36 37
   $pacemaker_managed = $lma_collector::params::pacemaker_managed,
37 38
   $rabbitmq_resource = undef,
38 39
   $aggregator_address = undef,
@@ -41,6 +42,7 @@ class lma_collector (
41 42
   include heka::params
42 43
 
43 44
   validate_hash($tags)
45
+  validate_integer($poolsize)
44 46
 
45 47
   $service_name = $lma_collector::params::service_name
46 48
   $config_dir = $lma_collector::params::config_dir
@@ -58,6 +60,7 @@ class lma_collector (
58 60
     max_message_size    => $lma_collector::params::hekad_max_message_size,
59 61
     max_process_inject  => $lma_collector::params::hekad_max_process_inject,
60 62
     max_timer_inject    => $lma_collector::params::hekad_max_timer_inject,
63
+    poolsize            => $poolsize,
61 64
   }
62 65
 
63 66
   if $pacemaker_managed {

Loading…
Cancel
Save