Enable HA on logging infrastructure

This commit aims to allow the loadbalancing for both
ElasticSearch and Kibana requests.

Change-Id: I6d17b62a8bbbe203dc24100b550bd716d23d2959

manifests/dashboard.pp

@@ -147,10 +147,6 @@ class cloud::dashboard(
   }
   $neutron_options_real = merge ($neutron_options, $neutron_extra_options)
 
-  ensure_resource('class', 'apache', {
-    default_vhost => false
-  })
-
   class { 'horizon':
     secret_key              => $secret_key,
     servername              => $servername,

manifests/database/nosql/elasticsearch.pp

@@ -0,0 +1,58 @@
+#
+# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# == Class: cloud::database::nosql::elasticsearch
+#
+# Install an ElasticSearch server
+#
+# === Parameters:
+#
+# [*listen_port*]
+#   (optional) Port on which ElasticSearch instance should listen
+#   Defaults to '9200'
+#
+# [*listen_ip*]
+#   (optional) IP address on which ElasticSearch instance should listen
+#   Defaults to '127.0.0.1'
+#
+# [*firewall_settings*]
+#   (optional) Allow to add custom parameters to firewall rules
+#   Should be an hash.
+#   Default to {}
+#
+class cloud::database::nosql::elasticsearch (
+  $listen_port       = '9200',
+  $listen_ip         = '127.0.0.1',
+  $firewall_settings = {},
+){
+
+  include ::elasticsearch
+
+  @@haproxy::balancermember{"${::fqdn}-es_cluster":
+    listening_service => 'elasticsearch',
+    server_names      => $::hostname,
+    ipaddresses       => $listen_ip,
+    ports             => $listen_port,
+    options           => 'check inter 2000 rise 2 fall 5'
+  }
+
+  if $::cloud::manage_firewall {
+    cloud::firewall::rule{ '100 allow elasticsearch access':
+      port   => $listen_port,
+      extras => $firewall_settings,
+    }
+  }
+
+}

manifests/loadbalancer.pp

@@ -164,6 +164,20 @@
 #   If set to false, no binding will be configure.
 #   Defaults to true
 #
+# [*elasticsearch*]
+#   (optional) Enable or not ElasticSearch binding.
+#   If true, both public and internal will attempt to be created except if vip_internal_ip is set to false.
+#   If set to ['10.0.0.1'], only IP in the array (or in the string) will be configured in the pool. They must be part of keepalived_ip options.
+#   If set to false, no binding will be configure.
+#   Defaults to true
+#
+# [*kibana*]
+#   (optional) Enable or not kibana binding.
+#   If true, both public and internal will attempt to be created except if vip_internal_ip is set to false.
+#   If set to ['10.0.0.1'], only IP in the array (or in the string) will be configured in the pool. They must be part of keepalived_ip options.
+#   If set to false, no binding will be configure.
+#   Defaults to true
+#
 # [*metadata_api*]
 #   (optional) Enable or not Metadata public binding.
 #   If true, both public and internal will attempt to be created except if vip_internal_ip is set to false.
@@ -304,6 +318,16 @@
 #   service configuration block.
 #   Defaults to []
 #
+# [*elasticsearch_bind_options*]
+#   (optional) A hash of options that are inserted into the HAproxy listening
+#   service configuration block.
+#   Defaults to []
+#
+# [*kibana_bind_options*]
+#   (optional) A hash of options that are inserted into the HAproxy listening
+#   service configuration block.
+#   Defaults to []
+#
 # [*galera_bind_options*]
 #   (optional) A hash of options that are inserted into the HAproxy listening
 #   service configuration block.
@@ -389,6 +413,14 @@
 #   (optional) Port of RabbitMQ service.
 #   Defaults to '5672'
 #
+# [*elasticsearch_port*]
+#   (optional) Port of ElasticSearch service.
+#   Defaults to '9200'
+#
+# [*kibana_port*]
+#   (optional) Port of Kibana service.
+#   Defaults to '8300'
+#
 # [*vip_public_ip*]
 #  (optional) Array or string for public VIP
 #  Should be part of keepalived_public_ips
@@ -437,6 +469,8 @@ class cloud::loadbalancer(
   $rabbitmq                         = false,
   $spice                            = false,
   $novnc                            = true,
+  $elasticsearch                    = true,
+  $kibana                           = true,
   $haproxy_auth                     = 'admin:changeme',
   $keepalived_state                 = 'BACKUP',
   $keepalived_priority              = '50',
@@ -468,6 +502,8 @@ class cloud::loadbalancer(
   $horizon_ssl_bind_options         = [],
   $rabbitmq_bind_options            = [],
   $galera_bind_options              = [],
+  $elasticsearch_bind_options       = [],
+  $kibana_bind_options              = [],
   $ks_ceilometer_public_port        = 8777,
   $ks_cinder_public_port            = 8776,
   $ks_ec2_public_port               = 8773,
@@ -488,6 +524,8 @@ class cloud::loadbalancer(
   $horizon_ssl_port                 = 443,
   $spice_port                       = 6082,
   $novnc_port                       = 6080,
+  $elasticsearch_port               = 9200,
+  $kibana_port                      = 8300,
   $vip_public_ip                    = ['127.0.0.1'],
   $vip_internal_ip                  = false,
   $vip_monitor_ip                   = false,
@@ -785,6 +823,19 @@ class cloud::loadbalancer(
     firewall_settings => $firewall_settings,
   }
 
+  cloud::loadbalancer::binding { 'elasticsearch':
+    ip                => $elasticsearch,
+    port              => $elasticsearch_port,
+    bind_options      => $elasticsearch_bind_options,
+    firewall_settings => $firewall_settings,
+  }
+  cloud::loadbalancer::binding { 'kibana':
+    ip                => $kibana,
+    port              => $kibana_port,
+    bind_options      => $kibana_bind_options,
+    firewall_settings => $firewall_settings,
+  }
+
   if (member(any2array($keepalived_public_ipvs), $galera_ip)) {
     warning('Exposing Galera cluster to public network is a security issue.')
   }

manifests/logging/server.pp

@@ -15,11 +15,46 @@
 #
 # == Class: cloud::logging::server
 #
-class cloud::logging::server{
+# [*kibana_port*]
+#   (optional) Port of Kibana service.
+#   Defaults to '8300'
+#
+# [*kibana_bind_ip*]
+#   (optional) Address on which kibana is listening on
+#   Defaults to '127.0.0.1'
+#
+# [*firewall_settings*]
+#   (optional) Allow to add custom parameters to firewall rules
+#   Should be an hash.
+#   Default to {}
+#
+class cloud::logging::server(
+  $kibana_port           = '8300',
+  $kibana_bind_ip        = '127.0.0.1',
+  $firewall_settings     = {},
+) {
+
+  Class['cloud::database::nosql::elasticsearch'] -> Class['kibana3']
+  Class['cloud::database::nosql::elasticsearch'] -> Class['cloud::logging::agent']
 
-  include ::elasticsearch
   include ::kibana3
+  include cloud::database::nosql::elasticsearch
   include cloud::logging::agent
   elasticsearch::instance {'fluentd' : }
 
+  @@haproxy::balancermember{"${::fqdn}-kibana":
+    listening_service => 'kibana',
+    server_names      => $::hostname,
+    ipaddresses       => $kibana_bind_ip,
+    ports             => $kibana_port,
+    options           => 'check inter 2000 rise 2 fall 5'
+  }
+
+  if $::cloud::manage_firewall {
+    cloud::firewall::rule{ '100 allow kibana access':
+      port   => $kibana_port,
+      extras => $firewall_settings,
+    }
+  }
+
 }

spec/classes/cloud_dashboard_spec.rb

@@ -38,6 +38,10 @@ describe 'cloud::dashboard' do
         :allowed_hosts              => 'horizon.openstack.org'}
     end
 
+    let :pre_condition do
+        "class { 'apache': default_vhost => false }"
+    end
+
     it 'configure horizon' do
       is_expected.to contain_class('horizon').with(
           :listen_ssl              => false,
@@ -122,7 +126,8 @@ describe 'cloud::dashboard' do
 
     context 'with default firewall enabled' do
       let :pre_condition do
-        "class { 'cloud': manage_firewall => true }"
+        "class { 'apache': default_vhost => false }
+         class { 'cloud': manage_firewall => true }"
       end
       it 'configure horizon firewall rules' do
         is_expected.to contain_firewall('100 allow horizon access').with(
@@ -135,7 +140,8 @@ describe 'cloud::dashboard' do
 
     context 'with custom firewall enabled' do
       let :pre_condition do
-        "class { 'cloud': manage_firewall => true }"
+        "class { 'apache': default_vhost => false }
+         class { 'cloud': manage_firewall => true }"
       end
       before :each do
         params.merge!(:firewall_settings => { 'limit' => '50/sec' } )

spec/classes/cloud_database_nosql_elasticsearch.rb

@@ -0,0 +1,81 @@
+#
+# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Unit tests for cloud::database::nosql::elasticsearch
+#
+
+require 'spec_helper'
+
+describe 'cloud::database::nosql::elasticsearch' do
+
+  shared_examples_for 'elasticsearch server' do
+
+    let :params do
+      { :firewall_settings => {} }
+    end
+
+    it 'configure elasticsearch' do
+      it is_expected.to contain_class('elasticsearch')
+    end
+
+    context 'with default firewall enabled' do
+      let :pre_condition do
+        "class { 'cloud': manage_firewall => true }"
+      end
+      it 'configure elasticsearch firewall rules' do
+        is_expected.to contain_firewall('100 allow elasticsearch access').with(
+          :port   => '9200',
+          :proto  => 'tcp',
+          :action => 'accept',
+        )
+      end
+    end
+
+    context 'with custom firewall enabled' do
+      let :pre_condition do
+        "class { 'cloud': manage_firewall => true }"
+      end
+      before :each do
+        params.merge!(:firewall_settings => { 'limit' => '50/sec' } )
+      end
+      it 'configure elasticsearch firewall rules with custom parameter' do
+        is_expected.to contain_firewall('100 allow elasticsearch access').with(
+          :port   => '9200',
+          :proto  => 'tcp',
+          :action => 'accept',
+          :limit  => '50/sec',
+        )
+      end
+    end
+
+  end
+
+  context 'on Debian platforms' do
+    let :facts do
+      { :osfamily => 'Debian' }
+    end
+
+    it_configures 'elasticsearch server'
+  end
+
+  context 'on RedHat platforms' do
+    let :facts do
+      { :osfamily => 'RedHat' }
+    end
+
+    it_configures 'elasticsearch server'
+  end
+
+end

spec/classes/cloud_logging_server.rb

@@ -1,59 +0,0 @@
-#
-# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-#
-# Unit tests for cloud::logging::server class
-#
-
-require 'spec_helper'
-
-describe 'cloud::logging::server' do
-
-  shared_examples_for 'openstack logging server' do
-
-    let :pre_condition do
-      "class { 'cloud::logging': }
-      include ::fluentd"
-    end
-
-    it 'configure logging common' do
-      it is_expected.to contain_concat("/etc/td-agent/config.d/forward.conf")
-    end
-
-    it 'configure kibana' do
-      is_expected.to contain_class('kibana3').with(:ws_port => '8001')
-    end
-
-    it 'configure an elasticsearch instance' do
-      is_expected.to contain_elasticsearch__instance('fluentd')
-    end
-  end
-
-  context 'on Debian platforms' do
-    let :facts do
-      { :osfamily => 'Debian' }
-    end
-
-    it_configures 'openstack logging server'
-  end
-
-  context 'on RedHat platforms' do
-    let :facts do
-      { :osfamily => 'RedHat' }
-    end
-
-    it_configures 'openstack logging server'
-  end
-
-end

spec/classes/cloud_logging_server_spec.rb

@@ -0,0 +1,101 @@
+#
+# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Unit tests for cloud::logging::server class
+#
+
+require 'spec_helper'
+
+describe 'cloud::logging::server' do
+
+  shared_examples_for 'openstack logging server' do
+
+    let :params do
+      { :firewall_settings => {} }
+    end
+
+    it 'configure kibana' do
+      is_expected.to contain_class('kibana3')
+    end
+
+    it 'configure the logging agent' do
+      is_expected.to contain_class('cloud::logging::agent')
+    end
+
+    it 'configure elasticsearch' do
+      is_expected.to contain_class('cloud::database::nosql::elasticsearch')
+    end
+
+    it 'configure an elasticsearch instance' do
+      is_expected.to contain_elasticsearch__instance('fluentd')
+    end
+
+    context 'with default firewall enabled' do
+      let :pre_condition do
+        "class { 'cloud': manage_firewall => true }"
+      end
+      it 'configure kibana firewall rules' do
+        is_expected.to contain_firewall('100 allow kibana access').with(
+          :port   => '8300',
+          :proto  => 'tcp',
+          :action => 'accept',
+        )
+      end
+    end
+
+    context 'with custom firewall enabled' do
+      let :pre_condition do
+        "class { 'cloud': manage_firewall => true }"
+      end
+      before :each do
+        params.merge!(:firewall_settings => { 'limit' => '50/sec' } )
+      end
+      it 'configure kibana firewall rules with custom parameter' do
+        is_expected.to contain_firewall('100 allow kibana access').with(
+          :port   => '8300',
+          :proto  => 'tcp',
+          :action => 'accept',
+          :limit  => '50/sec',
+        )
+      end
+    end
+
+  end
+
+  context 'on Debian platforms' do
+    let :facts do
+      {
+        :osfamily               => 'Debian',
+        :operatingsystem        => 'Debian',
+        :operatingsystemrelease => '7'
+       }
+    end
+
+    it_configures 'openstack logging server'
+  end
+
+  context 'on RedHat platforms' do
+    let :facts do
+      {
+        :osfamily               => 'RedHat',
+        :operatingsystem        => 'RedHat',
+        :operatingsystemrelease => '7'
+      }
+    end
+
+    it_configures 'openstack logging server'
+  end
+
+end