Merge "Use local ES balancers on logstash workers"

modules/elasticsearch/manifests/init.pp

@@ -20,15 +20,19 @@ class elasticsearch (
   $es_template_config = {}
 ) {
   # install java runtime
+  if ! defined(Package['openjdk-7-jre-headless']) {
     package { 'openjdk-7-jre-headless':
       ensure => present,
     }
+  }
 
   # Curl is handy for talking to the ES API on localhost. Allows for
   # querying cluster state and deleting indexes and so on.
+  if ! defined(Package['curl']) {
     package { 'curl':
       ensure => present,
     }
+  }
 
   exec { 'get_elasticsearch_deb':
     command => "wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${version}.deb -O /tmp/elasticsearch-${version}.deb",

modules/elasticsearch/templates/elasticsearch.yml.erb

@@ -86,6 +86,13 @@ node.name: "<%= scope.lookupvar("::hostname") %>"
 # to disable it, set the following:
 # node.max_local_storage_nodes: 1
 
+<% if es_template_config.has_key?('node.master') then -%>
+node.master: <%= es_template_config['node.master'] %>
+<% end -%>
+<% if es_template_config.has_key?('node.data') then -%>
+node.data: <%= es_template_config['node.data'] %>
+<% end -%>
+
 
 #################################### Index ####################################
 

modules/logstash/manifests/init.pp

@@ -75,7 +75,9 @@ class logstash {
     mode   => '0644',
   }
 
+  if ! defined(Package['openjdk-7-jre-headless']) {
     package { 'openjdk-7-jre-headless':
       ensure => present,
     }
+  }
 }

modules/logstash/manifests/watchdog.pp

@@ -23,9 +23,11 @@ class logstash::watchdog (
   package { 'jq':
     ensure => present,
   }
+  if ! defined(Package['curl']) {
     package { 'curl':
       ensure => present,
     }
+  }
 
   file { '/usr/local/bin/logstash-watchdog':
     ensure  => present,

modules/openstack_project/manifests/logstash_worker.pp

@@ -57,4 +57,19 @@ class openstack_project::logstash_worker (
   log_processor::worker { 'D':
     config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',
   }
+
+  class { '::elasticsearch':
+    es_template_config => {
+      'gateway.recover_after_nodes'          => '5',
+      'gateway.recover_after_time'           => '5m',
+      'gateway.expected_nodes'               => '6',
+      'discovery.zen.minimum_master_nodes'   => '5',
+      'discovery.zen.ping.multicast.enabled' => false,
+      'discovery.zen.ping.unicast.hosts'     => $elasticsearch_nodes,
+      'node.master'                          => false,
+      'node.data'                            => false,
+    },
+    heap_size          => '1g',
+    version            => '0.90.9',
+  }
 }

modules/openstack_project/templates/logstash/indexer.conf.erb

@@ -125,8 +125,8 @@ filter {
 
 output {
   elasticsearch_http {
-    host => "<%= scope.lookupvar("::openstack_project::logstash_worker::discover_node") %>"
+    host => "localhost"
     manage_template => false
-    flush_size => 512
+    flush_size => 1024
   }
 }