Initialize Elasticsearch data only once

All Elasticsearch templates and Kibana dashboard imports are performed only
once from the primary-elasticserach_kibana node.

Implements: blueprint elasticsearch-clustering

Change-Id: Ifa605c9dcdc603080b8adb80dbc71f2796cdf34b

deployment_scripts/puppet/manifests/elasticsearch.pp

@@ -75,12 +75,6 @@ elasticsearch::instance { $es_instance:
   }
 }
 
-lma_logging_analytics::es_template { ['log', 'notification']:
-  number_of_replicas => hiera('lma::elasticsearch::number_of_replicas'),
-  require            => Elasticsearch::Instance[$es_instance],
-  host               => $mgmt_address,
-}
-
 class { 'lma_logging_analytics::curator':
   retention_period => $elasticsearch_kibana['retention_period'],
   prefixes         => ['log', 'notification'],

deployment_scripts/puppet/manifests/kibana.pp

@@ -12,10 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
-prepare_network_config(hiera('network_scheme', {}))
-$mgmt_address = get_network_role_property('management', 'ipaddr')
 
 class { 'lma_logging_analytics::kibana':
-  number_of_replicas => hiera('lma::elasticsearch::number_of_replicas'),
-  es_host            => $mgmt_address,
+  es_host => hiera('lma::elasticsearch::vip'),
 }

deployment_scripts/puppet/manifests/provision_services.pp

@@ -0,0 +1,27 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+$vip = hiera('lma::elasticsearch::vip')
+$number_of_replicas = hiera('lma::elasticsearch::number_of_replicas')
+
+lma_logging_analytics::es_template { ['log', 'notification', 'kibana']:
+  number_of_replicas => $number_of_replicas,
+  host               => $vip,
+}
+
+lma_logging_analytics::kibana_dashboard { ['logs', 'notifications']:
+  host    => $vip,
+  require => Lma_logging_analytics::Es_template['kibana'],
+}

deployment_scripts/puppet/modules/lma_logging_analytics/manifests/kibana.pp

@@ -16,11 +16,10 @@
 
 
 class lma_logging_analytics::kibana (
-  $number_of_replicas = $lma_logging_analytics::params::kibana_replicas,
   $es_host = 'localhost',
-) inherits lma_logging_analytics::params {
+) {
 
-  validate_integer($number_of_replicas)
+  include lma_logging_analytics::params
 
   $kibana_dir    = $lma_logging_analytics::params::kibana_dir
   $kibana_conf   = $lma_logging_analytics::params::kibana_config
@@ -45,33 +44,11 @@ class lma_logging_analytics::kibana (
     require => File[$kibana_dir],
   }
 
-  elasticsearch::template { 'kibana':
-    content => "{\"template\":\"kibana-*\", \"settings\": {\"number_of_replicas\":${number_of_replicas}}}",
-    host    => $es_host,
-  }
-
-  # Note that the dashboards are stored in templates/ because it is the only way
-  # for us to read the content of the file. Ideally we would have used the
-  # file() function but until Puppet 3.7 it works only with absolute paths.
-  # See http://www.unixdaemon.net/tools/puppet/puppet-3.7-file-function.html
-  # for details
-  lma_logging_analytics::kibana_dashboard { 'logs':
-    content => template('lma_logging_analytics/kibana_dashboards/logs.json'),
-    host    => $es_host,
-    require => [File["${dashboard_dir}/logs.json"], Elasticsearch::Template['kibana']],
-  }
-
   file { "${dashboard_dir}/notifications.json":
     content => template('lma_logging_analytics/kibana_dashboards/notifications.json'),
     require => File[$kibana_dir],
   }
 
-  lma_logging_analytics::kibana_dashboard { 'notifications':
-    content => template('lma_logging_analytics/kibana_dashboards/notifications.json'),
-    host    => $es_host,
-    require => [File["${dashboard_dir}/notifications.json"], Elasticsearch::Template['kibana']],
-  }
-
   # Install nginx
   class { 'nginx':
     manage_repo           => false,

deployment_scripts/puppet/modules/lma_logging_analytics/manifests/kibana_dashboard.pp

@@ -15,7 +15,6 @@
 define lma_logging_analytics::kibana_dashboard (
   $host = 'localhost',
   $port = '9200',
-  $content = undef,
 ) {
   include lma_logging_analytics::params
 
@@ -23,6 +22,13 @@ define lma_logging_analytics::kibana_dashboard (
   $dashboard_title = join([$lma_logging_analytics::params::kibana_dashboard_prefix, capitalize($title)], '')
   $dashboard_id = uriescape($dashboard_title)
 
+  # Note that the dashboards are stored in templates/ because it is the only way
+  # for us to read the content of the file. Ideally we would have used the
+  # file() function but until Puppet 3.7 it only works with absolute paths.
+  # See http://www.unixdaemon.net/tools/puppet/puppet-3.7-file-function.html
+  # for details
+  $content = template("lma_logging_analytics/kibana_dashboards/${title}.json")
+
   $dashboard_source = encode_kibana_dashboard('guest', 'guest', $dashboard_title, $content)
 
   exec { $title:

deployment_scripts/puppet/modules/lma_logging_analytics/spec/classes/lma_logging_analytics_kibana_spec.rb

@@ -23,8 +23,5 @@ describe 'lma_logging_analytics::kibana' do
     it { should create_file('/opt/kibana/config.js')}
     it { should create_file('/opt/kibana/app/dashboards/logs.json')}
     it { should create_file('/opt/kibana/app/dashboards/notifications.json')}
-    it { should create_elasticsearch__template('kibana')}
-    it { should create_lma_logging_analytics__kibana_dashboard('logs')}
-    it { should create_lma_logging_analytics__kibana_dashboard('notifications')}
     it { should create_class('nginx')}
 end

deployment_scripts/puppet/modules/lma_logging_analytics/spec/defines/lma_logging_analytics_kibana_dashboard_spec.rb

@@ -14,13 +14,13 @@
 require 'spec_helper'
 
 describe 'lma_logging_analytics::kibana_dashboard' do
-    let(:title) { 'foo' }
+    let(:title) { 'logs' }
     let(:facts) do
         {:kernel => 'Linux', :operatingsystem => 'Ubuntu',
          :concat_basedir => '/foo'}
     end
 
     describe 'with defaults' do
-        it { is_expected.to contain_exec('foo') }
+        it { is_expected.to contain_exec('logs') }
     end
 end

deployment_scripts/puppet/modules/lma_logging_analytics/templates/es_template_kibana.json.erb

@@ -0,0 +1,7 @@
+{
+    "settings": {
+        "number_of_shards": <%= @number_of_shards %>,
+        "number_of_replicas": <%= @number_of_replicas %>
+    },
+    "template": "<%= @index_prefix %>-*"
+}

tasks.yaml

@@ -23,3 +23,10 @@
     puppet_modules: puppet/modules
     timeout: 1200
 
+- role: ['primary-elasticsearch_kibana']
+  stage: post_deployment/8100
+  type: puppet
+  parameters:
+    puppet_manifest: puppet/manifests/provision_services.pp
+    puppet_modules: puppet/modules
+    timeout: 600