opendev/puppet-openstackci
Code Issues Proposed changes

Merge "Add generic logstash manifests"

Browse Source
This commit is contained in:
Jenkins
2015-11-18 00:23:44 +00:00
committed by Gerrit Code Review
parent 3a42ad03d2 95146ff74c
commit 3890e2311f
5 changed files with 233 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
manifests/elasticsearch_node.pp Normal file
View File

@@ -0,0 +1,59 @@
# Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# == Class: openstackci::elasticsearch_node
# Elasticsearch server glue class.
#
class openstackci::elasticsearch_node (
$discover_nodes = ['localhost'],
$es_heap_size = '30g',
$es_version = '0.90.9',
$es_gw_recover_after_nodes = '5',
$es_gw_recover_after_time = '5m',
$es_gw_expected_nodes = '6',
$es_discovery_min_master_nodes = '4',
$es_indices_cleanup_hour = '2',
$es_indices_cleanup_minute = '0',
$es_indices_cleanup_period = '10 days ago',
) {
class { 'logstash::elasticsearch': }
class { '::elasticsearch':
es_template_config => {
'index.store.compress.stored' => true,
'index.store.compress.tv' => true,
'indices.memory.index_buffer_size' => '33%',
'bootstrap.mlockall' => true,
'gateway.recover_after_nodes' => $es_gw_recover_after_nodes,
'gateway.recover_after_time' => $es_gw_recover_after_time,
'gateway.expected_nodes' => $es_gw_expected_nodes,
'discovery.zen.minimum_master_nodes' => $es_discovery_min_master_nodes,
'discovery.zen.ping.multicast.enabled' => false,
'discovery.zen.ping.unicast.hosts' => $discover_nodes,
},
heap_size => $es_heap_size,
version => $es_version,
}
cron { 'delete_old_es_indices':
user => 'root',
hour => $es_indices_cleanup_hour,
minute => $es_indices_cleanup_minute,
command => "curl -sS -XDELETE \"http://localhost:9200/logstash-`date -d '${es_indices_cleanup_period}' +\%Y.\%m.\%d`/\" > /dev/null",
environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin',
}
}

53
manifests/logstash.pp Normal file
View File

@@ -0,0 +1,53 @@
# Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# == Class: openstackci::logstash
# Logstash web frontend glue class.
#
class openstackci::logstash (
$discover_nodes = ['localhost'],
$statsd_host,
$subunit2sql_db_host,
$subunit2sql_db_pass,
$log_processor_config,
) {
class { 'logstash::web':
frontend => 'kibana',
discover_nodes => $discover_nodes,
proxy_elasticsearch => true,
}
class { 'log_processor': }
class { 'log_processor::client':
config_file => $log_processor_config,
statsd_host => $statsd_host,
}
include subunit2sql
class { 'subunit2sql::server':
db_host => $subunit2sql_db_host,
db_pass => $subunit2sql_db_pass,
}
include simpleproxy
class { 'simpleproxy::server':
db_host => $subunit2sql_db_host,
}
}

67
manifests/logstash_worker.pp Normal file
View File

@@ -0,0 +1,67 @@
# Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# == Class: openstackci::logstash_worker
#
# Logstash indexer worker glue class.
#
class openstackci::logstash_worker (
$elasticsearch_nodes = [],
$log_processor_config,
$es_heap_size = '1g',
$es_version = '0.90.9',
# Increase the indexer max heap size to twice the default.
# Default is 25% of memory or 1g whichever is less.
$indexer_java_args = '-Xmx2g',
$indexer_conf_template,
$log_processor_workers = ['A', 'B', 'C', 'D',],
$es_gw_recover_after_nodes = '5',
$es_gw_recover_after_time = '5m',
$es_gw_expected_nodes = '6',
$es_discovery_min_master_nodes = '5',
) {
file { '/etc/default/logstash-indexer':
ensure => present,
owner => 'root',
group => 'root',
mode => '0644',
content => "JAVA_ARGS='${indexer_java_args}'"
}
class { 'logstash::indexer':
conf_template => $indexer_conf_template,
}
include log_processor
log_processor::worker { $log_processor_workers:
config_file => $log_processor_config,
}
class { '::elasticsearch':
es_template_config => {
'gateway.recover_after_nodes' => $es_gw_recover_after_nodes,
'gateway.recover_after_time' => $es_gw_recover_after_time,
'gateway.expected_nodes' => $es_gw_expected_nodes,
'discovery.zen.minimum_master_nodes' => $es_discovery_min_master_nodes,
'discovery.zen.ping.multicast.enabled' => false,
'discovery.zen.ping.unicast.hosts' => $elasticsearch_nodes,
'node.master' => false,
'node.data' => false,
},
heap_size => $es_heap_size,
version => $es_version,
}
}

48
manifests/subunit_worker.pp Normal file
View File

@@ -0,0 +1,48 @@
# Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# == Class: openstackci::subunit_worker
#
# subunit2sql worker glue class.
#
class openstackci::subunit_worker (
$subunit2sql_db_host,
$subunit2sql_db_pass,
$subunit2sql_config_file,
) {
include subunit2sql
subunit2sql::worker { 'A':
config_file => $subunit2sql_config_file,
db_host => $subunit2sql_db_host,
db_pass => $subunit2sql_db_pass,
require => [
User['logstash'],
Group['logstash'],
]
}
# [mmedvede]: subunit2sql is supposed to create users it uses,
# but it doesn't yet.
if ! defined(User['logstash']) {
user {'logstash':
ensure => present,
}
}
if ! defined(Group['logstash']) {
group {'logstash':
ensure => present,
}
}
}

7
metadata.json
View File

@@ -12,6 +12,11 @@
{ "name": "openstackinfra/vcsrepo" },
{ "name": "openstackinfra/jenkins" },
{ "name": "openstackinfra/zuul" },
{ "name": "openstackinfra/project_config" }
{ "name": "openstackinfra/project_config" },
{ "name": "openstackinfra/log_processor" },
{ "name": "openstackinfra/simpleproxy" },
{ "name": "openstackinfra/subunit2sql" },
{ "name": "openstackinfra/logstash" },
{ "name": "openstackinfra/elasticsearch" }
]
}