From 1c35a4eda8d1a10411835bd156588324b4e1140a Mon Sep 17 00:00:00 2001
From: Simon Pasquier <spasquier@mirantis.com>
Date: Mon, 9 Jan 2017 17:20:20 +0100
Subject: [PATCH] Get rid of auto_expand_replicas for Kibana indices

Setting auto_expand_replicas to "0-all" can make the Kibana indices
unavailable when scaling up the Elasticsearch cluster. As a consequence,
the Kibana service is unavailable and the operator needs to manually fix
the problem. This change applies the same replication settings as for
the log and notification indices leading to more predictable behavior.
It also enforces the replication settings in the provision_services.pp
manifest to deal with scale-down and scale-up operations.

Change-Id: I8979f3d006ccd908711cbe0862032dc7b73d9b62
Closes-Bug: #1552258
---
 .../puppet/manifests/kibana_index_configuration.pp     | 10 ++++++----
 .../puppet/manifests/provision_services.pp             |  7 +++++++
 .../templates/es_template_kibana4.json.erb             |  3 +--
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/deployment_scripts/puppet/manifests/kibana_index_configuration.pp b/deployment_scripts/puppet/manifests/kibana_index_configuration.pp
index 57f0f95..919ef3a 100644
--- a/deployment_scripts/puppet/manifests/kibana_index_configuration.pp
+++ b/deployment_scripts/puppet/manifests/kibana_index_configuration.pp
@@ -17,6 +17,7 @@ notice('fuel-plugin-elasticsearch-kibana: kibana_index_configuration.pp')
 $vip = hiera('lma::elasticsearch::vip')
 $es_port = hiera('lma::elasticsearch::rest_port')
 $kibana_index = hiera('lma::elasticsearch::kibana_index')
+$number_of_replicas = hiera('lma::elasticsearch::number_of_replicas')
 
 # Elasticsearch must be reachable through HAproxy before the template creation.
 # This is due the fact that The Elasticsearch Puppet module miserably fails
@@ -34,10 +35,11 @@ haproxy_backend_status { 'elasticsearch':
 # Then, correct fields mapping are present before importing objects
 # (searches, visualizations and dashboards).
 lma_logging_analytics::es_template { 'kibana4':
-  host           => $vip,
-  port           => $es_port,
-  index_template => $kibana_index,
-  require        => Haproxy_backend_status['elasticsearch'],
+  number_of_replicas => $number_of_replicas,
+  host               => $vip,
+  port               => $es_port,
+  index_template     => $kibana_index,
+  require            => Haproxy_backend_status['elasticsearch'],
 }
 
 # Import all Kibana objects in one time by issuing a Bulk request
diff --git a/deployment_scripts/puppet/manifests/provision_services.pp b/deployment_scripts/puppet/manifests/provision_services.pp
index 0cb803c..0f2395e 100644
--- a/deployment_scripts/puppet/manifests/provision_services.pp
+++ b/deployment_scripts/puppet/manifests/provision_services.pp
@@ -21,6 +21,7 @@ $kibana_vip = hiera('lma::kibana::vip')
 $kibana_viewer_port = hiera('lma::elasticsearch::kibana_frontend_viewer_port')
 $es_port = hiera('lma::elasticsearch::rest_port')
 $number_of_replicas = hiera('lma::elasticsearch::number_of_replicas')
+$kibana_index = hiera('lma::elasticsearch::kibana_index')
 
 $authnz = hiera_hash('lma::kibana::authnz')
 if $authnz['ldap_enabled'] and $authnz['ldap_authorization_enabled'] {
@@ -66,6 +67,12 @@ lma_logging_analytics::es_template { ['log', 'notification']:
   port               => $es_port,
 }
 
+# Adjust the number of replicas for the Kibana index
+exec { 'adjust_kibana_replicas':
+  command => "/usr/bin/curl -sL -XPUT http://${es_vip}:${es_port}/${kibana_index}/_settings \
+  -d '{\"index\": {\"number_of_replicas\": ${number_of_replicas}}}'"
+}
+
 $kibana_link_created_file = '/var/cache/kibana_link_created'
 exec { 'notify_kibana_url':
   creates => $kibana_link_created_file,
diff --git a/deployment_scripts/puppet/modules/lma_logging_analytics/templates/es_template_kibana4.json.erb b/deployment_scripts/puppet/modules/lma_logging_analytics/templates/es_template_kibana4.json.erb
index fb78f72..a102a46 100644
--- a/deployment_scripts/puppet/modules/lma_logging_analytics/templates/es_template_kibana4.json.erb
+++ b/deployment_scripts/puppet/modules/lma_logging_analytics/templates/es_template_kibana4.json.erb
@@ -132,8 +132,7 @@
     },
     "settings": {
         "number_of_shards": <%= @number_of_shards %>,
-        "auto_expand_replicas": "0-all"
+        "number_of_replicas": <%= @number_of_replicas %>
     },
     "template": "<%= @template %>"
 }
-