opendev/system-config
Code Issues Proposed changes

Add two more elasticsearch nodes.

Browse Source 
* manifests/site.pp: List the two new elasticsearch nodes in the
appropriate lists.

* modules/logstash/manifests/elasticsearch.pp: Do not restart
elasticsearch when config files change. Service restarts are costly and
should be manually performed when necessary. Otherwise puppet should
simply update the config files.

* modules/logstash/templates/elasticsearch.yml.erb: Update elasticsearch
config with new cluster topology. Increase memory available for
indexing.

* modules/openstack_project/manifests/cacti.pp: Add new nodes to cacti
monitoring list.

Adding two more elasticsearch nodes to relieve memory pressure (more
nodes means fewer indexes per nodes which requires less memory to
manage). And two more nodes gives us more disk to retain older indexes
in. These new nodes should allow us to retain at least 3 weeks of
indexed logs.

Change-Id: I3a5a02311e939c8147e401110c7b96d085eb8274
Reviewed-on: https://review.openstack.org/36305
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: Clark Boylan <clark.boylan@gmail.com>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins
This commit is contained in:
Clark Boylan 2013-07-09 10:40:54 -07:00 committed by Jenkins
parent f7d8a94095
commit 76d3e632d4
4 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
manifests/site.pp
View File

@ -206,6 +206,8 @@ node 'logstash.openstack.org' {
'elasticsearch.openstack.org',
'elasticsearch2.openstack.org',
'elasticsearch3.openstack.org',
'elasticsearch4.openstack.org',
'elasticsearch5.openstack.org',
],
gearman_workers => [
'logstash-worker1.openstack.org',
@ -218,6 +220,8 @@ node 'logstash.openstack.org' {
'elasticsearch.openstack.org:9200',
'elasticsearch2.openstack.org:9200',
'elasticsearch3.openstack.org:9200',
'elasticsearch4.openstack.org:9200',
'elasticsearch5.openstack.org:9200',
],
}
}
@ -229,6 +233,8 @@ node /^logstash-worker\d+\.openstack\.org$/ {
'elasticsearch.openstack.org',
'elasticsearch2.openstack.org',
'elasticsearch3.openstack.org',
'elasticsearch4.openstack.org',
'elasticsearch5.openstack.org',
],
discover_node => 'elasticsearch.openstack.org',
}
@ -241,6 +247,8 @@ node /^elasticsearch\d*\.openstack\.org$/ {
'elasticsearch.openstack.org',
'elasticsearch2.openstack.org',
'elasticsearch3.openstack.org',
'elasticsearch4.openstack.org',
'elasticsearch5.openstack.org',
],
elasticsearch_clients => [
'logstash.openstack.org',
@ -254,6 +262,8 @@ node /^elasticsearch\d*\.openstack\.org$/ {
'elasticsearch.openstack.org',
'elasticsearch2.openstack.org',
'elasticsearch3.openstack.org',
'elasticsearch4.openstack.org',
'elasticsearch5.openstack.org',
],
}
}

4
modules/logstash/manifests/elasticsearch.pp
View File

@ -72,8 +72,8 @@ class logstash::elasticsearch (
service { 'elasticsearch':
ensure => running,
require => Package['elasticsearch'],
subscribe => [
require => [
Package['elasticsearch'],
File['/etc/elasticsearch/elasticsearch.yml'],
File['/etc/elasticsearch/default-mapping.json'],
File['/etc/default/elasticsearch'],

6
modules/logstash/templates/elasticsearch.yml.erb
View File

@ -139,7 +139,7 @@ node.name: "<%= scope.lookupvar("::hostname") %>"
index.store.compress.stored: true
index.store.compress.tv: true
indices.memory.index_buffer_size: "33%"
indices.memory.index_buffer_size: "40%"
#################################### Paths ####################################
@ -255,7 +255,7 @@ bootstrap.mlockall: true
# Allow recovery process after N nodes in a cluster are up:
#
gateway.recover_after_nodes: 2
gateway.recover_after_nodes: 4
# Set the timeout to initiate the recovery process, once the N nodes
# from previous setting are up (accepts time value):
@ -266,7 +266,7 @@ gateway.recover_after_time: 5m
# are up (and recover_after_nodes is met), begin recovery process immediately
# (without waiting for recover_after_time to expire):
#
gateway.expected_nodes: 3
gateway.expected_nodes: 5
############################# Recovery Throttling #############################

2
modules/openstack_project/manifests/cacti.pp
View File

@ -62,6 +62,8 @@ class openstack_project::cacti (
'elasticsearch.openstack.org',
'elasticsearch2.openstack.org',
'elasticsearch3.openstack.org',
'elasticsearch4.openstack.org',
'elasticsearch5.openstack.org',
'etherpad.openstack.org',
'graphite.openstack.org',
'jenkins.openstack.org',