opendev/system-config
Code Issues Proposed changes

Better elasticsearch cluster settings.

Browse Source 
Use mutliple discover nodes to determine elasticsearch cluster
membership. Put a timeout on recovery starting instead of the default
to recovery immediately. Describe cluster topology in elasticsearch yaml
config so that it can make smarter decisions. Round robin kibana
requests across each discover node.

Change-Id: I08ef9dd158ddf6a6ce01dfb2050626f543d45b10
Reviewed-on: https://review.openstack.org/34106
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Reviewed-by: James E. Blair <corvus@inaugust.com>
Approved: James E. Blair <corvus@inaugust.com>
Tested-by: Jenkins
This commit is contained in:
Clark Boylan 2013-06-22 10:39:04 -07:00 committed by Jenkins
parent cfd1df90d9
commit 443539b7fa
10 changed files with 29 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
manifests/site.pp
View File

@ -214,7 +214,11 @@ node 'logstash.openstack.org' {
'logstash-worker4.openstack.org', 'logstash-worker4.openstack.org',
'logstash-worker5.openstack.org', 'logstash-worker5.openstack.org',
], ],
discover_node => 'elasticsearch.openstack.org', discover_nodes => [
'elasticsearch.openstack.org:9200',
'elasticsearch2.openstack.org:9200',
'elasticsearch3.openstack.org:9200',
],
} }
} }
@ -246,7 +250,11 @@ node /^elasticsearch\d*\.openstack\.org$/ {
'logstash-worker4.openstack.org', 'logstash-worker4.openstack.org',
'logstash-worker5.openstack.org', 'logstash-worker5.openstack.org',
], ],
discover_node => 'elasticsearch.openstack.org', discover_nodes => [
'elasticsearch.openstack.org',
'elasticsearch2.openstack.org',
'elasticsearch3.openstack.org',
],
} }
} }

2
modules/kibana/manifests/init.pp
View File

@ -15,7 +15,7 @@
# Class to install kibana frontend to logstash. # Class to install kibana frontend to logstash.
# #
class kibana ( class kibana (
$discover_node = 'localhost' $discover_nodes = ['localhost:9200']
) { ) {
group { 'kibana': group { 'kibana':

2
modules/kibana/templates/config.rb.erb
View File

@ -7,7 +7,7 @@ module KibanaConfig
# Your elastic search server(s). This may be set as an array for round robin # Your elastic search server(s). This may be set as an array for round robin
# load balancing # load balancing
# Elasticsearch = ["elasticsearch1:9200","elasticsearch2:9200"] # Elasticsearch = ["elasticsearch1:9200","elasticsearch2:9200"]
Elasticsearch = "<%= scope.lookupvar("::kibana::discover_node") %>:9200" Elasticsearch = ["<%= scope.lookupvar("::kibana::discover_nodes").join("\",\"") %>"]
#Set the Net::HTTP read/open timeouts for the connection to the ES backend #Set the Net::HTTP read/open timeouts for the connection to the ES backend
ElasticsearchTimeout = 500 ElasticsearchTimeout = 500

2
modules/logstash/files/elasticsearch.default
View File

@ -17,7 +17,7 @@ ES_HEAP_SIZE=16g
# Maximum locked memory size. Set to "unlimited" if you use the # Maximum locked memory size. Set to "unlimited" if you use the
# bootstrap.mlockall option in elasticsearch.yml. You must also set # bootstrap.mlockall option in elasticsearch.yml. You must also set
# ES_HEAP_SIZE. # ES_HEAP_SIZE.
#MAX_LOCKED_MEMORY=unlimited MAX_LOCKED_MEMORY=unlimited
# ElasticSearch log directory # ElasticSearch log directory
#LOG_DIR=/var/log/elasticsearch #LOG_DIR=/var/log/elasticsearch

2
modules/logstash/manifests/elasticsearch.pp
View File

@ -15,7 +15,7 @@
# Class to install elasticsearch. # Class to install elasticsearch.
# #
class logstash::elasticsearch ( class logstash::elasticsearch (
discover_node = 'localhost' discover_nodes = ['localhost']
) { ) {
# install java runtime # install java runtime
package { 'java7-runtime-headless': package { 'java7-runtime-headless':

4
modules/logstash/manifests/web.pp
View File

@ -18,7 +18,7 @@ class logstash::web (
$vhost_name = $::fqdn, $vhost_name = $::fqdn,
$serveradmin = "webmaster@${::fqdn}", $serveradmin = "webmaster@${::fqdn}",
$frontend = 'internal', $frontend = 'internal',
$discover_node = 'localhost', $discover_nodes = ['localhost:9200'],
$proxy_elasticsearch = false $proxy_elasticsearch = false
) { ) {
include apache include apache
@ -57,7 +57,7 @@ class logstash::web (
'kibana': { 'kibana': {
class { 'kibana': class { 'kibana':
discover_node => $discover_node, discover_nodes => $discover_nodes,
} }
$vhost = 'logstash/kibana.vhost.erb' $vhost = 'logstash/kibana.vhost.erb'
} }

12
modules/logstash/templates/elasticsearch.yml.erb
View File

@ -255,18 +255,18 @@ bootstrap.mlockall: true
# Allow recovery process after N nodes in a cluster are up: # Allow recovery process after N nodes in a cluster are up:
# #
# gateway.recover_after_nodes: 1 gateway.recover_after_nodes: 2
# Set the timeout to initiate the recovery process, once the N nodes # Set the timeout to initiate the recovery process, once the N nodes
# from previous setting are up (accepts time value): # from previous setting are up (accepts time value):
# #
# gateway.recover_after_time: 5m gateway.recover_after_time: 5m
# Set how many nodes are expected in this cluster. Once these N nodes # Set how many nodes are expected in this cluster. Once these N nodes
# are up (and recover_after_nodes is met), begin recovery process immediately # are up (and recover_after_nodes is met), begin recovery process immediately
# (without waiting for recover_after_time to expire): # (without waiting for recover_after_time to expire):
# #
# gateway.expected_nodes: 2 gateway.expected_nodes: 3
############################# Recovery Throttling ############################# ############################# Recovery Throttling #############################
@ -304,7 +304,7 @@ bootstrap.mlockall: true
# operational within the cluster. Set this option to a higher value (2-4) # operational within the cluster. Set this option to a higher value (2-4)
# for large clusters (>3 nodes): # for large clusters (>3 nodes):
# #
# discovery.zen.minimum_master_nodes: 1 discovery.zen.minimum_master_nodes: 2
# Set the time to wait for ping responses from other nodes when discovering. # Set the time to wait for ping responses from other nodes when discovering.
# Set this option to a higher value on a slow or congested network # Set this option to a higher value on a slow or congested network
@ -327,7 +327,9 @@ discovery.zen.ping.multicast.enabled: false
# to perform discovery when new nodes (master or data) are started: # to perform discovery when new nodes (master or data) are started:
# #
# discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"] # discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"]
discovery.zen.ping.unicast.hosts: ["<%= discover_node %>"] #
# Make all master eligible nodes discover nodes.
discovery.zen.ping.unicast.hosts: ["<%= discover_nodes.join("\", \"") %>"]
# EC2 discovery allows to use AWS EC2 API in order to perform discovery. # EC2 discovery allows to use AWS EC2 API in order to perform discovery.
# #

4
modules/logstash/templates/kibana.vhost.erb
View File

@ -11,9 +11,9 @@
<% if proxy_elasticsearch == true %> <% if proxy_elasticsearch == true %>
# Proxy for elasticsearch _aliases, .*/_status, and .*/_search. # Proxy for elasticsearch _aliases, .*/_status, and .*/_search.
<LocationMatch "^/elasticsearch/(_aliases|.*/_status|.*/_search)$"> <LocationMatch "^/elasticsearch/(_aliases|.*/_status|.*/_search)$">
ProxyPassMatch http://<%= scope.lookupvar("::logstash::web::discover_node") %>:9200/$1 ProxyPassMatch http://<%= scope.lookupvar("::logstash::web::discover_nodes")[0] %>/$1
</LocationMatch> </LocationMatch>
ProxyPassReverse /elasticsearch/ http://<%= scope.lookupvar("::logstash::web::discover_node") %>:9200/ ProxyPassReverse /elasticsearch/ http://<%= scope.lookupvar("::logstash::web::discover_nodes")[0] %>/
<% end %> <% end %>
ProxyPass / http://127.0.0.1:5601/ retry=0 ProxyPass / http://127.0.0.1:5601/ retry=0

4
modules/openstack_project/manifests/elasticsearch.pp
View File

@ -17,7 +17,7 @@
class openstack_project::elasticsearch ( class openstack_project::elasticsearch (
$elasticsearch_nodes = [], $elasticsearch_nodes = [],
$elasticsearch_clients = [], $elasticsearch_clients = [],
$discover_node = 'localhost', $discover_nodes = ['localhost'],
$sysadmins = [] $sysadmins = []
) { ) {
$iptables_nodes_rule = regsubst ($elasticsearch_nodes, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') $iptables_nodes_rule = regsubst ($elasticsearch_nodes, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT')
@ -31,7 +31,7 @@ class openstack_project::elasticsearch (
} }
class { 'logstash::elasticsearch': class { 'logstash::elasticsearch':
discover_node => $discover_node, discover_nodes => $discover_nodes,
} }
cron { 'delete_old_es_indices': cron { 'delete_old_es_indices':

4
modules/openstack_project/manifests/logstash.pp
View File

@ -17,7 +17,7 @@
class openstack_project::logstash ( class openstack_project::logstash (
$elasticsearch_nodes = [], $elasticsearch_nodes = [],
$gearman_workers = [], $gearman_workers = [],
$discover_node = 'elasticsearch.openstack.org', $discover_nodes = ['elasticsearch.openstack.org:9200'],
$sysadmins = [] $sysadmins = []
) { ) {
$iptables_es_rule = regsubst ($elasticsearch_nodes, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') $iptables_es_rule = regsubst ($elasticsearch_nodes, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT')
@ -32,7 +32,7 @@ class openstack_project::logstash (
class { 'logstash::web': class { 'logstash::web':
frontend => 'kibana', frontend => 'kibana',
discover_node => $discover_node, discover_nodes => $discover_nodes,
proxy_elasticsearch => true, proxy_elasticsearch => true,
} }