stackforge/puppet-openstack-cloud
Code Issues Proposed changes

Enable HA on logging infrastructure

Browse Source 
This commit aims to allow the loadbalancing for both
ElasticSearch and Kibana requests.

Change-Id: I6d17b62a8bbbe203dc24100b550bd716d23d2959
This commit is contained in:
Yanis Guenane
2015-02-02 07:42:45 -05:00
parent a54f3e61ea
commit 7bef2787ce
8 changed files with 336 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
manifests/dashboard.pp
View File

@@ -147,10 +147,6 @@ class cloud::dashboard(
}
$neutron_options_real = merge ($neutron_options, $neutron_extra_options)
ensure_resource('class', 'apache', {
default_vhost => false
})
class { 'horizon':
secret_key => $secret_key,
servername => $servername,

58
manifests/database/nosql/elasticsearch.pp Normal file
View File

@@ -0,0 +1,58 @@
#
# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# == Class: cloud::database::nosql::elasticsearch
#
# Install an ElasticSearch server
#
# === Parameters:
#
# [*listen_port*]
# (optional) Port on which ElasticSearch instance should listen
# Defaults to '9200'
#
# [*listen_ip*]
# (optional) IP address on which ElasticSearch instance should listen
# Defaults to '127.0.0.1'
#
# [*firewall_settings*]
# (optional) Allow to add custom parameters to firewall rules
# Should be an hash.
# Default to {}
#
class cloud::database::nosql::elasticsearch (
$listen_port = '9200',
$listen_ip = '127.0.0.1',
$firewall_settings = {},
){
include ::elasticsearch
@@haproxy::balancermember{"${::fqdn}-es_cluster":
listening_service => 'elasticsearch',
server_names => $::hostname,
ipaddresses => $listen_ip,
ports => $listen_port,
options => 'check inter 2000 rise 2 fall 5'
}
if $::cloud::manage_firewall {
cloud::firewall::rule{ '100 allow elasticsearch access':
port => $listen_port,
extras => $firewall_settings,
}
}
}

51
manifests/loadbalancer.pp
View File

@@ -164,6 +164,20 @@
# If set to false, no binding will be configure.
# Defaults to true
#
# [*elasticsearch*]
# (optional) Enable or not ElasticSearch binding.
# If true, both public and internal will attempt to be created except if vip_internal_ip is set to false.
# If set to ['10.0.0.1'], only IP in the array (or in the string) will be configured in the pool. They must be part of keepalived_ip options.
# If set to false, no binding will be configure.
# Defaults to true
#
# [*kibana*]
# (optional) Enable or not kibana binding.
# If true, both public and internal will attempt to be created except if vip_internal_ip is set to false.
# If set to ['10.0.0.1'], only IP in the array (or in the string) will be configured in the pool. They must be part of keepalived_ip options.
# If set to false, no binding will be configure.
# Defaults to true
#
# [*metadata_api*]
# (optional) Enable or not Metadata public binding.
# If true, both public and internal will attempt to be created except if vip_internal_ip is set to false.
@@ -304,6 +318,16 @@
# service configuration block.
# Defaults to []
#
# [*elasticsearch_bind_options*]
# (optional) A hash of options that are inserted into the HAproxy listening
# service configuration block.
# Defaults to []
#
# [*kibana_bind_options*]
# (optional) A hash of options that are inserted into the HAproxy listening
# service configuration block.
# Defaults to []
#
# [*galera_bind_options*]
# (optional) A hash of options that are inserted into the HAproxy listening
# service configuration block.
@@ -389,6 +413,14 @@
# (optional) Port of RabbitMQ service.
# Defaults to '5672'
#
# [*elasticsearch_port*]
# (optional) Port of ElasticSearch service.
# Defaults to '9200'
#
# [*kibana_port*]
# (optional) Port of Kibana service.
# Defaults to '8300'
#
# [*vip_public_ip*]
# (optional) Array or string for public VIP
# Should be part of keepalived_public_ips
@@ -437,6 +469,8 @@ class cloud::loadbalancer(
$rabbitmq = false,
$spice = false,
$novnc = true,
$elasticsearch = true,
$kibana = true,
$haproxy_auth = 'admin:changeme',
$keepalived_state = 'BACKUP',
$keepalived_priority = '50',
@@ -468,6 +502,8 @@ class cloud::loadbalancer(
$horizon_ssl_bind_options = [],
$rabbitmq_bind_options = [],
$galera_bind_options = [],
$elasticsearch_bind_options = [],
$kibana_bind_options = [],
$ks_ceilometer_public_port = 8777,
$ks_cinder_public_port = 8776,
$ks_ec2_public_port = 8773,
@@ -488,6 +524,8 @@ class cloud::loadbalancer(
$horizon_ssl_port = 443,
$spice_port = 6082,
$novnc_port = 6080,
$elasticsearch_port = 9200,
$kibana_port = 8300,
$vip_public_ip = ['127.0.0.1'],
$vip_internal_ip = false,
$vip_monitor_ip = false,
@@ -785,6 +823,19 @@ class cloud::loadbalancer(
firewall_settings => $firewall_settings,
}
cloud::loadbalancer::binding { 'elasticsearch':
ip => $elasticsearch,
port => $elasticsearch_port,
bind_options => $elasticsearch_bind_options,
firewall_settings => $firewall_settings,
}
cloud::loadbalancer::binding { 'kibana':
ip => $kibana,
port => $kibana_port,
bind_options => $kibana_bind_options,
firewall_settings => $firewall_settings,
}
if (member(any2array($keepalived_public_ipvs), $galera_ip)) {
warning('Exposing Galera cluster to public network is a security issue.')
}

39
manifests/logging/server.pp
View File

@@ -15,11 +15,46 @@
#
# == Class: cloud::logging::server
#
class cloud::logging::server{
# [*kibana_port*]
# (optional) Port of Kibana service.
# Defaults to '8300'
#
# [*kibana_bind_ip*]
# (optional) Address on which kibana is listening on
# Defaults to '127.0.0.1'
#
# [*firewall_settings*]
# (optional) Allow to add custom parameters to firewall rules
# Should be an hash.
# Default to {}
#
class cloud::logging::server(
$kibana_port = '8300',
$kibana_bind_ip = '127.0.0.1',
$firewall_settings = {},
) {
Class['cloud::database::nosql::elasticsearch'] -> Class['kibana3']
Class['cloud::database::nosql::elasticsearch'] -> Class['cloud::logging::agent']
include ::elasticsearch
include ::kibana3
include cloud::database::nosql::elasticsearch
include cloud::logging::agent
elasticsearch::instance {'fluentd' : }
@@haproxy::balancermember{"${::fqdn}-kibana":
listening_service => 'kibana',
server_names => $::hostname,
ipaddresses => $kibana_bind_ip,
ports => $kibana_port,
options => 'check inter 2000 rise 2 fall 5'
}
if $::cloud::manage_firewall {
cloud::firewall::rule{ '100 allow kibana access':
port => $kibana_port,
extras => $firewall_settings,
}
}
}

10
spec/classes/cloud_dashboard_spec.rb
View File

@@ -38,6 +38,10 @@ describe 'cloud::dashboard' do
:allowed_hosts => 'horizon.openstack.org'}
end
let :pre_condition do
"class { 'apache': default_vhost => false }"
end
it 'configure horizon' do
is_expected.to contain_class('horizon').with(
:listen_ssl => false,
@@ -122,7 +126,8 @@ describe 'cloud::dashboard' do
context 'with default firewall enabled' do
let :pre_condition do
"class { 'cloud': manage_firewall => true }"
"class { 'apache': default_vhost => false }
class { 'cloud': manage_firewall => true }"
end
it 'configure horizon firewall rules' do
is_expected.to contain_firewall('100 allow horizon access').with(
@@ -135,7 +140,8 @@ describe 'cloud::dashboard' do
context 'with custom firewall enabled' do
let :pre_condition do
"class { 'cloud': manage_firewall => true }"
"class { 'apache': default_vhost => false }
class { 'cloud': manage_firewall => true }"
end
before :each do
params.merge!(:firewall_settings => { 'limit' => '50/sec' } )

81
spec/classes/cloud_database_nosql_elasticsearch.rb Normal file
View File

@@ -0,0 +1,81 @@
#
# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Unit tests for cloud::database::nosql::elasticsearch
#
require 'spec_helper'
describe 'cloud::database::nosql::elasticsearch' do
shared_examples_for 'elasticsearch server' do
let :params do
{ :firewall_settings => {} }
end
it 'configure elasticsearch' do
it is_expected.to contain_class('elasticsearch')
end
context 'with default firewall enabled' do
let :pre_condition do
"class { 'cloud': manage_firewall => true }"
end
it 'configure elasticsearch firewall rules' do
is_expected.to contain_firewall('100 allow elasticsearch access').with(
:port => '9200',
:proto => 'tcp',
:action => 'accept',
)
end
end
context 'with custom firewall enabled' do
let :pre_condition do
"class { 'cloud': manage_firewall => true }"
end
before :each do
params.merge!(:firewall_settings => { 'limit' => '50/sec' } )
end
it 'configure elasticsearch firewall rules with custom parameter' do
is_expected.to contain_firewall('100 allow elasticsearch access').with(
:port => '9200',
:proto => 'tcp',
:action => 'accept',
:limit => '50/sec',
)
end
end
end
context 'on Debian platforms' do
let :facts do
{ :osfamily => 'Debian' }
end
it_configures 'elasticsearch server'
end
context 'on RedHat platforms' do
let :facts do
{ :osfamily => 'RedHat' }
end
it_configures 'elasticsearch server'
end
end

59
spec/classes/cloud_logging_server.rb
View File

@@ -1,59 +0,0 @@
#
# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Unit tests for cloud::logging::server class
#
require 'spec_helper'
describe 'cloud::logging::server' do
shared_examples_for 'openstack logging server' do
let :pre_condition do
"class { 'cloud::logging': }
include ::fluentd"
end
it 'configure logging common' do
it is_expected.to contain_concat("/etc/td-agent/config.d/forward.conf")
end
it 'configure kibana' do
is_expected.to contain_class('kibana3').with(:ws_port => '8001')
end
it 'configure an elasticsearch instance' do
is_expected.to contain_elasticsearch__instance('fluentd')
end
end
context 'on Debian platforms' do
let :facts do
{ :osfamily => 'Debian' }
end
it_configures 'openstack logging server'
end
context 'on RedHat platforms' do
let :facts do
{ :osfamily => 'RedHat' }
end
it_configures 'openstack logging server'
end
end

101
spec/classes/cloud_logging_server_spec.rb Normal file
View File

@@ -0,0 +1,101 @@
#
# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Unit tests for cloud::logging::server class
#
require 'spec_helper'
describe 'cloud::logging::server' do
shared_examples_for 'openstack logging server' do
let :params do
{ :firewall_settings => {} }
end
it 'configure kibana' do
is_expected.to contain_class('kibana3')
end
it 'configure the logging agent' do
is_expected.to contain_class('cloud::logging::agent')
end
it 'configure elasticsearch' do
is_expected.to contain_class('cloud::database::nosql::elasticsearch')
end
it 'configure an elasticsearch instance' do
is_expected.to contain_elasticsearch__instance('fluentd')
end
context 'with default firewall enabled' do
let :pre_condition do
"class { 'cloud': manage_firewall => true }"
end
it 'configure kibana firewall rules' do
is_expected.to contain_firewall('100 allow kibana access').with(
:port => '8300',
:proto => 'tcp',
:action => 'accept',
)
end
end
context 'with custom firewall enabled' do
let :pre_condition do
"class { 'cloud': manage_firewall => true }"
end
before :each do
params.merge!(:firewall_settings => { 'limit' => '50/sec' } )
end
it 'configure kibana firewall rules with custom parameter' do
is_expected.to contain_firewall('100 allow kibana access').with(
:port => '8300',
:proto => 'tcp',
:action => 'accept',
:limit => '50/sec',
)
end
end
end
context 'on Debian platforms' do
let :facts do
{
:osfamily => 'Debian',
:operatingsystem => 'Debian',
:operatingsystemrelease => '7'
}
end
it_configures 'openstack logging server'
end
context 'on RedHat platforms' do
let :facts do
{
:osfamily => 'RedHat',
:operatingsystem => 'RedHat',
:operatingsystemrelease => '7'
}
end
it_configures 'openstack logging server'
end
end