Better elasticsearch cluster settings.

Use mutliple discover nodes to determine elasticsearch cluster
membership. Put a timeout on recovery starting instead of the default
to recovery immediately. Describe cluster topology in elasticsearch yaml
config so that it can make smarter decisions. Round robin kibana
requests across each discover node.

Change-Id: I08ef9dd158ddf6a6ce01dfb2050626f543d45b10
Reviewed-on: https://review.openstack.org/34106
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Reviewed-by: James E. Blair <corvus@inaugust.com>
Approved: James E. Blair <corvus@inaugust.com>
Tested-by: Jenkins

files/elasticsearch.default

@@ -17,7 +17,7 @@ ES_HEAP_SIZE=16g
 # Maximum locked memory size. Set to "unlimited" if you use the
 # bootstrap.mlockall option in elasticsearch.yml. You must also set
 # ES_HEAP_SIZE.
-#MAX_LOCKED_MEMORY=unlimited
+MAX_LOCKED_MEMORY=unlimited
 
 # ElasticSearch log directory
 #LOG_DIR=/var/log/elasticsearch

manifests/elasticsearch.pp

@@ -15,7 +15,7 @@
 # Class to install elasticsearch.
 #
 class logstash::elasticsearch (
-  discover_node = 'localhost'
+  discover_nodes = ['localhost']
 ) {
   # install java runtime
   package { 'java7-runtime-headless':

manifests/web.pp

@@ -18,7 +18,7 @@ class logstash::web (
   $vhost_name = $::fqdn,
   $serveradmin = "webmaster@${::fqdn}",
   $frontend = 'internal',
-  $discover_node = 'localhost',
+  $discover_nodes = ['localhost:9200'],
   $proxy_elasticsearch = false
 ) {
   include apache
@@ -57,7 +57,7 @@ class logstash::web (
 
     'kibana': {
       class { 'kibana':
-        discover_node => $discover_node,
+        discover_nodes => $discover_nodes,
       }
       $vhost = 'logstash/kibana.vhost.erb'
     }

templates/elasticsearch.yml.erb

@@ -255,18 +255,18 @@ bootstrap.mlockall: true
 
 # Allow recovery process after N nodes in a cluster are up:
 #
-# gateway.recover_after_nodes: 1
+gateway.recover_after_nodes: 2
 
 # Set the timeout to initiate the recovery process, once the N nodes
 # from previous setting are up (accepts time value):
 #
-# gateway.recover_after_time: 5m
+gateway.recover_after_time: 5m
 
 # Set how many nodes are expected in this cluster. Once these N nodes
 # are up (and recover_after_nodes is met), begin recovery process immediately
 # (without waiting for recover_after_time to expire):
 #
-# gateway.expected_nodes: 2
+gateway.expected_nodes: 3
 
 
 ############################# Recovery Throttling #############################
@@ -304,7 +304,7 @@ bootstrap.mlockall: true
 # operational within the cluster. Set this option to a higher value (2-4)
 # for large clusters (>3 nodes):
 #
-# discovery.zen.minimum_master_nodes: 1
+discovery.zen.minimum_master_nodes: 2
 
 # Set the time to wait for ping responses from other nodes when discovering.
 # Set this option to a higher value on a slow or congested network
@@ -327,7 +327,9 @@ discovery.zen.ping.multicast.enabled: false
 #    to perform discovery when new nodes (master or data) are started:
 #
 # discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"]
-discovery.zen.ping.unicast.hosts: ["<%= discover_node %>"]
+#
+# Make all master eligible nodes discover nodes.
+discovery.zen.ping.unicast.hosts: ["<%= discover_nodes.join("\", \"") %>"]
 
 # EC2 discovery allows to use AWS EC2 API in order to perform discovery.
 #

templates/kibana.vhost.erb

@@ -11,9 +11,9 @@
              <% if proxy_elasticsearch == true %>
              # Proxy for elasticsearch _aliases, .*/_status, and .*/_search.
              <LocationMatch "^/elasticsearch/(_aliases|.*/_status|.*/_search)$">
-               ProxyPassMatch http://<%= scope.lookupvar("::logstash::web::discover_node") %>:9200/$1
+               ProxyPassMatch http://<%= scope.lookupvar("::logstash::web::discover_nodes")[0] %>/$1
              </LocationMatch>
-             ProxyPassReverse /elasticsearch/ http://<%= scope.lookupvar("::logstash::web::discover_node") %>:9200/
+             ProxyPassReverse /elasticsearch/ http://<%= scope.lookupvar("::logstash::web::discover_nodes")[0] %>/
              <% end %>
 
              ProxyPass / http://127.0.0.1:5601/ retry=0