opendev/system-config
Code Issues Proposed changes

Cleanup elasticsearch related config

Browse Source 
Remove elasticsearch01 as it has been turned off.

Allow all elasticsearch running nodes to talk to each other over ports
9200 to 9400. This includes the logstash-worker nodes running non data
elasticsearch clients. This appears to be necessary to keep logs from
filling up with gigs of NoRouteToHost exceptions on the worker nodes.

Change-Id: I3e3365a717d22d881654206692d17dd05ed32c98
This commit is contained in:
Clark Boylan 2014-08-06 17:56:16 -07:00
parent 764ee8c74f
commit 38b53a4bee
2 changed files with 34 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
manifests/site.pp
View File

@ -6,7 +6,6 @@
# passed around in test.sh
#
$elasticsearch_nodes = [
'elasticsearch01.openstack.org',
'elasticsearch02.openstack.org',
'elasticsearch03.openstack.org',
'elasticsearch04.openstack.org',
@ -14,6 +13,29 @@ $elasticsearch_nodes = [
'elasticsearch06.openstack.org',
'elasticsearch07.openstack.org',
]
$elasticsearch_clients = [
'logstash.openstack.org',
'logstash-worker01.openstack.org',
'logstash-worker02.openstack.org',
'logstash-worker03.openstack.org',
'logstash-worker04.openstack.org',
'logstash-worker05.openstack.org',
'logstash-worker06.openstack.org',
'logstash-worker07.openstack.org',
'logstash-worker08.openstack.org',
'logstash-worker09.openstack.org',
'logstash-worker10.openstack.org',
'logstash-worker11.openstack.org',
'logstash-worker12.openstack.org',
'logstash-worker13.openstack.org',
'logstash-worker14.openstack.org',
'logstash-worker15.openstack.org',
'logstash-worker16.openstack.org',
'logstash-worker17.openstack.org',
'logstash-worker18.openstack.org',
'logstash-worker19.openstack.org',
'logstash-worker20.openstack.org',
]
#
# Default: should at least behave like an openstack server
@ -290,31 +312,10 @@ node 'wiki.openstack.org' {
# Node-OS: precise
node 'logstash.openstack.org' {
class { 'openstack_project::logstash':
sysadmins => hiera('sysadmins', []),
elasticsearch_nodes => $elasticsearch_nodes,
gearman_workers => [
'logstash-worker01.openstack.org',
'logstash-worker02.openstack.org',
'logstash-worker03.openstack.org',
'logstash-worker04.openstack.org',
'logstash-worker05.openstack.org',
'logstash-worker06.openstack.org',
'logstash-worker07.openstack.org',
'logstash-worker08.openstack.org',
'logstash-worker09.openstack.org',
'logstash-worker10.openstack.org',
'logstash-worker11.openstack.org',
'logstash-worker12.openstack.org',
'logstash-worker13.openstack.org',
'logstash-worker14.openstack.org',
'logstash-worker15.openstack.org',
'logstash-worker16.openstack.org',
'logstash-worker17.openstack.org',
'logstash-worker18.openstack.org',
'logstash-worker19.openstack.org',
'logstash-worker20.openstack.org',
],
discover_nodes => [
sysadmins => hiera('sysadmins', []),
elasticsearch_nodes => $elasticsearch_nodes,
gearman_workers => $elasticsearch_clients,
discover_nodes => [
'elasticsearch02.openstack.org:9200',
'elasticsearch03.openstack.org:9200',
'elasticsearch04.openstack.org:9200',
@ -328,9 +329,10 @@ node 'logstash.openstack.org' {
# Node-OS: precise
node /^logstash-worker\d+\.openstack\.org$/ {
class { 'openstack_project::logstash_worker':
sysadmins => hiera('sysadmins', []),
elasticsearch_nodes => $elasticsearch_nodes,
discover_node => 'elasticsearch02.openstack.org',
sysadmins => hiera('sysadmins', []),
elasticsearch_nodes => $elasticsearch_nodes,
elasticsearch_clients => $elasticsearch_clients,
discover_node => 'elasticsearch02.openstack.org',
}
}
@ -339,29 +341,7 @@ node /^elasticsearch0[1-7]\.openstack\.org$/ {
class { 'openstack_project::elasticsearch_node':
sysadmins => hiera('sysadmins', []),
elasticsearch_nodes => $elasticsearch_nodes,
elasticsearch_clients => [
'logstash.openstack.org',
'logstash-worker01.openstack.org',
'logstash-worker02.openstack.org',
'logstash-worker03.openstack.org',
'logstash-worker04.openstack.org',
'logstash-worker05.openstack.org',
'logstash-worker06.openstack.org',
'logstash-worker07.openstack.org',
'logstash-worker08.openstack.org',
'logstash-worker09.openstack.org',
'logstash-worker10.openstack.org',
'logstash-worker11.openstack.org',
'logstash-worker12.openstack.org',
'logstash-worker13.openstack.org',
'logstash-worker14.openstack.org',
'logstash-worker15.openstack.org',
'logstash-worker16.openstack.org',
'logstash-worker17.openstack.org',
'logstash-worker18.openstack.org',
'logstash-worker19.openstack.org',
'logstash-worker20.openstack.org',
],
elasticsearch_clients => $elasticsearch_clients,
discover_nodes => $elasticsearch_nodes,
}
}

3
modules/openstack_project/manifests/logstash_worker.pp
View File

@ -16,10 +16,11 @@
#
class openstack_project::logstash_worker (
$elasticsearch_nodes = [],
$elasticsearch_clients = [],
$discover_node = 'elasticsearch01.openstack.org',
$sysadmins = []
) {
$iptables_rule = regsubst ($elasticsearch_nodes, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT')
$iptables_rule = regsubst (flatten([$elasticsearch_nodes, $elasticsearch_clients]) , '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT')
class { 'openstack_project::server':
iptables_public_tcp_ports => [22],
iptables_rules6 => $iptables_rule,