From 76d3e632d4fdcf5ec19b1845ec654bddbfa311b9 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Tue, 9 Jul 2013 10:40:54 -0700
Subject: [PATCH] Add two more elasticsearch nodes.

* manifests/site.pp: List the two new elasticsearch nodes in the
appropriate lists.

* modules/logstash/manifests/elasticsearch.pp: Do not restart
elasticsearch when config files change. Service restarts are costly and
should be manually performed when necessary. Otherwise puppet should
simply update the config files.

* modules/logstash/templates/elasticsearch.yml.erb: Update elasticsearch
config with new cluster topology. Increase memory available for
indexing.

* modules/openstack_project/manifests/cacti.pp: Add new nodes to cacti
monitoring list.

Adding two more elasticsearch nodes to relieve memory pressure (more
nodes means fewer indexes per nodes which requires less memory to
manage). And two more nodes gives us more disk to retain older indexes
in. These new nodes should allow us to retain at least 3 weeks of
indexed logs.

Change-Id: I3a5a02311e939c8147e401110c7b96d085eb8274
Reviewed-on: https://review.openstack.org/36305
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: Clark Boylan <clark.boylan@gmail.com>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins
---
 manifests/site.pp                                | 10 ++++++++++
 modules/logstash/manifests/elasticsearch.pp      |  4 ++--
 modules/logstash/templates/elasticsearch.yml.erb |  6 +++---
 modules/openstack_project/manifests/cacti.pp     |  2 ++
 4 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/manifests/site.pp b/manifests/site.pp
index 1eabc5705a..cf50e6cee1 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -206,6 +206,8 @@ node 'logstash.openstack.org' {
       'elasticsearch.openstack.org',
       'elasticsearch2.openstack.org',
       'elasticsearch3.openstack.org',
+      'elasticsearch4.openstack.org',
+      'elasticsearch5.openstack.org',
     ],
     gearman_workers     => [
       'logstash-worker1.openstack.org',
@@ -218,6 +220,8 @@ node 'logstash.openstack.org' {
       'elasticsearch.openstack.org:9200',
       'elasticsearch2.openstack.org:9200',
       'elasticsearch3.openstack.org:9200',
+      'elasticsearch4.openstack.org:9200',
+      'elasticsearch5.openstack.org:9200',
     ],
   }
 }
@@ -229,6 +233,8 @@ node /^logstash-worker\d+\.openstack\.org$/ {
       'elasticsearch.openstack.org',
       'elasticsearch2.openstack.org',
       'elasticsearch3.openstack.org',
+      'elasticsearch4.openstack.org',
+      'elasticsearch5.openstack.org',
     ],
     discover_node       => 'elasticsearch.openstack.org',
   }
@@ -241,6 +247,8 @@ node /^elasticsearch\d*\.openstack\.org$/ {
       'elasticsearch.openstack.org',
       'elasticsearch2.openstack.org',
       'elasticsearch3.openstack.org',
+      'elasticsearch4.openstack.org',
+      'elasticsearch5.openstack.org',
     ],
     elasticsearch_clients => [
       'logstash.openstack.org',
@@ -254,6 +262,8 @@ node /^elasticsearch\d*\.openstack\.org$/ {
       'elasticsearch.openstack.org',
       'elasticsearch2.openstack.org',
       'elasticsearch3.openstack.org',
+      'elasticsearch4.openstack.org',
+      'elasticsearch5.openstack.org',
     ],
   }
 }
diff --git a/modules/logstash/manifests/elasticsearch.pp b/modules/logstash/manifests/elasticsearch.pp
index 106a25478a..f4b2410fe6 100644
--- a/modules/logstash/manifests/elasticsearch.pp
+++ b/modules/logstash/manifests/elasticsearch.pp
@@ -72,8 +72,8 @@ class logstash::elasticsearch (
 
   service { 'elasticsearch':
     ensure    => running,
-    require   => Package['elasticsearch'],
-    subscribe => [
+    require   => [
+      Package['elasticsearch'],
       File['/etc/elasticsearch/elasticsearch.yml'],
       File['/etc/elasticsearch/default-mapping.json'],
       File['/etc/default/elasticsearch'],
diff --git a/modules/logstash/templates/elasticsearch.yml.erb b/modules/logstash/templates/elasticsearch.yml.erb
index 6e102ec75b..4e82837d36 100644
--- a/modules/logstash/templates/elasticsearch.yml.erb
+++ b/modules/logstash/templates/elasticsearch.yml.erb
@@ -139,7 +139,7 @@ node.name: "<%= scope.lookupvar("::hostname") %>"
 index.store.compress.stored: true
 index.store.compress.tv: true
 
-indices.memory.index_buffer_size: "33%"
+indices.memory.index_buffer_size: "40%"
 
 #################################### Paths ####################################
 
@@ -255,7 +255,7 @@ bootstrap.mlockall: true
 
 # Allow recovery process after N nodes in a cluster are up:
 #
-gateway.recover_after_nodes: 2
+gateway.recover_after_nodes: 4
 
 # Set the timeout to initiate the recovery process, once the N nodes
 # from previous setting are up (accepts time value):
@@ -266,7 +266,7 @@ gateway.recover_after_time: 5m
 # are up (and recover_after_nodes is met), begin recovery process immediately
 # (without waiting for recover_after_time to expire):
 #
-gateway.expected_nodes: 3
+gateway.expected_nodes: 5
 
 
 ############################# Recovery Throttling #############################
diff --git a/modules/openstack_project/manifests/cacti.pp b/modules/openstack_project/manifests/cacti.pp
index d858d2b3c1..044df6c7b7 100644
--- a/modules/openstack_project/manifests/cacti.pp
+++ b/modules/openstack_project/manifests/cacti.pp
@@ -62,6 +62,8 @@ class openstack_project::cacti (
     'elasticsearch.openstack.org',
     'elasticsearch2.openstack.org',
     'elasticsearch3.openstack.org',
+    'elasticsearch4.openstack.org',
+    'elasticsearch5.openstack.org',
     'etherpad.openstack.org',
     'graphite.openstack.org',
     'jenkins.openstack.org',