From 5bbe7824e0a2cbe866c65358cf68ae1649b6df55 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Sun, 26 May 2013 16:08:46 -0700 Subject: [PATCH] Scale out logstash indexing to multiple hosts. Logstash performs filtering in a single thread so it does not scale up very well. Work around this by scaling Logstash out to multiple indexer hosts. Current plan is to have a small (2GB) kibana web front end host that does nothing but talk to elasticsearch, three 4GB logstash indexers that will run a single log-pusher.py + logstash indexer with some partition of the logfiles assigned to each indexer, and finally the existing large elasticsearch node. Eventually properly load balancing log processing across the worker nodes would be great, but the current partition method should work well enough with little additional effort. Change-Id: Ifc6396560934314ffd6a7c47eb2acff9e9c2a7af Reviewed-on: https://review.openstack.org/30573 Reviewed-by: James E. Blair Approved: Jeremy Stanley Reviewed-by: Jeremy Stanley Tested-by: Jenkins --- manifests/site.pp | 23 +++++- .../logstash-worker1/jenkins-log-pusher.yaml | 29 +++++++ .../logstash-worker2/jenkins-log-pusher.yaml | 42 ++++++++++ .../jenkins-log-pusher.yaml | 38 --------- .../manifests/elasticsearch.pp | 3 +- .../openstack_project/manifests/jenkins.pp | 3 +- .../openstack_project/manifests/logstash.pp | 57 +------------ .../manifests/logstash_worker.pp | 81 +++++++++++++++++++ 8 files changed, 180 insertions(+), 96 deletions(-) create mode 100644 modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml create mode 100644 modules/openstack_project/files/logstash/logstash-worker2/jenkins-log-pusher.yaml rename modules/openstack_project/files/logstash/{ => logstash-worker3}/jenkins-log-pusher.yaml (53%) create mode 100644 modules/openstack_project/manifests/logstash_worker.pp diff --git a/manifests/site.pp b/manifests/site.pp index eee1b297e0..ccd2c72934 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -75,6 +75,11 @@ node 'jenkins.openstack.org' { ssl_key_file_contents => hiera('jenkins_ssl_key_file_contents'), ssl_chain_file_contents => hiera('jenkins_ssl_chain_file_contents'), sysadmins => hiera('sysadmins'), + zmq_event_receivers => [ + 'logstash-worker1.openstack.org', + 'logstash-worker2.openstack.org', + 'logstash-worker3.openstack.org', + ], } } @@ -200,13 +205,27 @@ node 'puppet-dashboard.openstack.org' { node 'logstash.openstack.org' { class { 'openstack_project::logstash': - sysadmins => hiera('sysadmins'), + sysadmins => hiera('sysadmins'), + elasticsearch_masters => ['elasticsearch.openstack.org'], + } +} + +node /^logstash-worker\d+\.openstack\.org$/ { + class { 'openstack_project::logstash_worker': + sysadmins => hiera('sysadmins'), + elasticsearch_masters => ['elasticsearch.openstack.org'], } } node 'elasticsearch.openstack.org' { class { 'openstack_project::elasticsearch': - sysadmins => hiera('sysadmins'), + sysadmins => hiera('sysadmins'), + logstash_workers => [ + 'logstash.openstack.org', + 'logstash-worker1.openstack.org', + 'logstash-worker2.openstack.org', + 'logstash-worker3.openstack.org', + ], } } diff --git a/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml b/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml new file mode 100644 index 0000000000..5be9257ddb --- /dev/null +++ b/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml @@ -0,0 +1,29 @@ +# Defaults +source-defaults: + source-url: http://logs.openstack.org + output-host: localhost + output-port: 9999 + output-mode: tcp + retry-get: False + +# List of zmq event inputs. +zmq-publishers: + - tcp://jenkins.openstack.org:8888 + +# List of files to source logs from. +source-files: + - name: console.html + retry-get: True + - name: logs/screen-c-api.txt + tags: + - screen + - oslofmt + - name: logs/screen-c-sch.txt + tags: + - screen + - oslofmt + - name: logs/screen-c-vol.txt + tags: + - screen + - oslofmt +# TODO(clarkb) Add keystone, swift, and syslog logs here. diff --git a/modules/openstack_project/files/logstash/logstash-worker2/jenkins-log-pusher.yaml b/modules/openstack_project/files/logstash/logstash-worker2/jenkins-log-pusher.yaml new file mode 100644 index 0000000000..8becc1523f --- /dev/null +++ b/modules/openstack_project/files/logstash/logstash-worker2/jenkins-log-pusher.yaml @@ -0,0 +1,42 @@ +# Defaults +source-defaults: + source-url: http://logs.openstack.org + output-host: localhost + output-port: 9999 + output-mode: tcp + retry-get: False + +# List of zmq event inputs. +zmq-publishers: + - tcp://jenkins.openstack.org:8888 + +# List of files to source logs from. +source-files: + - name: logs/screen-n-api.txt + tags: + - screen + - oslofmt + - name: logs/screen-n-cond.txt + tags: + - screen + - oslofmt + - name: logs/screen-n-cpu.txt + tags: + - screen + - oslofmt + - name: logs/screen-n-crt.txt + tags: + - screen + - oslofmt + - name: logs/screen-n-net.txt + tags: + - screen + - oslofmt + - name: logs/screen-n-obj.txt + tags: + - screen + - oslofmt + - name: logs/screen-n-sch.txt + tags: + - screen + - oslofmt diff --git a/modules/openstack_project/files/logstash/jenkins-log-pusher.yaml b/modules/openstack_project/files/logstash/logstash-worker3/jenkins-log-pusher.yaml similarity index 53% rename from modules/openstack_project/files/logstash/jenkins-log-pusher.yaml rename to modules/openstack_project/files/logstash/logstash-worker3/jenkins-log-pusher.yaml index 5ee455ca6a..497826112e 100644 --- a/modules/openstack_project/files/logstash/jenkins-log-pusher.yaml +++ b/modules/openstack_project/files/logstash/logstash-worker3/jenkins-log-pusher.yaml @@ -12,20 +12,6 @@ zmq-publishers: # List of files to source logs from. source-files: - - name: console.html - retry-get: True - - name: logs/screen-c-api.txt - tags: - - screen - - oslofmt - - name: logs/screen-c-sch.txt - tags: - - screen - - oslofmt - - name: logs/screen-c-vol.txt - tags: - - screen - - oslofmt - name: logs/screen-g-api.txt tags: - screen @@ -34,30 +20,6 @@ source-files: tags: - screen - oslofmt - - name: logs/screen-n-api.txt - tags: - - screen - - oslofmt - - name: logs/screen-n-cond.txt - tags: - - screen - - oslofmt - - name: logs/screen-n-cpu.txt - tags: - - screen - - oslofmt - - name: logs/screen-n-crt.txt - tags: - - screen - - oslofmt - - name: logs/screen-n-obj.txt - tags: - - screen - - oslofmt - - name: logs/screen-n-sch.txt - tags: - - screen - - oslofmt - name: logs/screen-q-agt.txt tags: - screen diff --git a/modules/openstack_project/manifests/elasticsearch.pp b/modules/openstack_project/manifests/elasticsearch.pp index 59afcd8d5a..420eba6253 100644 --- a/modules/openstack_project/manifests/elasticsearch.pp +++ b/modules/openstack_project/manifests/elasticsearch.pp @@ -15,9 +15,10 @@ # Elasticsearch server glue class. # class openstack_project::elasticsearch ( + $logstash_workers = [], $sysadmins = [] ) { - $iptables_rule = '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s logstash.openstack.org -j ACCEPT' + $iptables_rule = regsubst ($logstash_workers, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') class { 'openstack_project::server': iptables_public_tcp_ports => [22], iptables_rules6 => $iptables_rule, diff --git a/modules/openstack_project/manifests/jenkins.pp b/modules/openstack_project/manifests/jenkins.pp index 8d3c597b26..40df0cfe5c 100644 --- a/modules/openstack_project/manifests/jenkins.pp +++ b/modules/openstack_project/manifests/jenkins.pp @@ -7,11 +7,12 @@ class openstack_project::jenkins ( $ssl_key_file_contents = '', $ssl_chain_file_contents = '', $jenkins_ssh_private_key = '', + $zmq_event_receivers = [], $sysadmins = [] ) { include openstack_project - $iptables_rule = '-m state --state NEW -m tcp -p tcp --dport 8888 -s logstash.openstack.org -j ACCEPT' + $iptables_rule = regsubst ($zmq_event_receivers, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 8888 -s \1 -j ACCEPT') class { 'openstack_project::server': iptables_public_tcp_ports => [80, 443], iptables_rules6 => $iptables_rule, diff --git a/modules/openstack_project/manifests/logstash.pp b/modules/openstack_project/manifests/logstash.pp index 87197b2a17..760e51ea0b 100644 --- a/modules/openstack_project/manifests/logstash.pp +++ b/modules/openstack_project/manifests/logstash.pp @@ -12,12 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. # -# Logstash indexer server glue class. +# Logstash web frontend glue class. # class openstack_project::logstash ( + $elasticsearch_masters = [], $sysadmins = [] ) { - $iptables_rule = '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s elasticsearch.openstack.org -j ACCEPT' + $iptables_rule = regsubst ($elasticsearch_masters, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') class { 'openstack_project::server': iptables_public_tcp_ports => [22, 80], iptables_rules6 => $iptables_rule, @@ -25,60 +26,8 @@ class openstack_project::logstash ( sysadmins => $sysadmins, } - class { 'logstash::indexer': - conf_template => 'openstack_project/logstash/indexer.conf.erb', - } class { 'logstash::web': frontend => 'kibana', elasticsearch_host => 'elasticsearch.openstack.org', } - - package { 'python3': - ensure => 'present', - } - - package { 'python3-zmq': - ensure => 'present', - } - - package { 'python3-yaml': - ensure => 'present', - } - - file { '/usr/local/bin/log-pusher.py': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/logstash/log-pusher.py', - require => Package['python3'], - } - - file { '/etc/logstash/jenkins-log-pusher.yaml': - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - source => 'puppet:///modules/openstack_project/logstash/jenkins-log-pusher.yaml', - require => Class['logstash::indexer'], - } - - file { '/etc/init.d/jenkins-log-pusher': - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - source => 'puppet:///modules/openstack_project/logstash/jenkins-log-pusher.init', - require => [ - File['/usr/local/bin/log-pusher.py'], - File['/etc/logstash/jenkins-log-pusher.yaml'], - ], - } - - service { 'jenkins-log-pusher': - enable => true, - hasrestart => true, - subscribe => File['/etc/logstash/jenkins-log-pusher.yaml'], - require => File['/etc/init.d/jenkins-log-pusher'], - } } diff --git a/modules/openstack_project/manifests/logstash_worker.pp b/modules/openstack_project/manifests/logstash_worker.pp new file mode 100644 index 0000000000..f6ee930cde --- /dev/null +++ b/modules/openstack_project/manifests/logstash_worker.pp @@ -0,0 +1,81 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Logstash indexer worker glue class. +# +class openstack_project::logstash_worker ( + $elasticsearch_masters = [], + $sysadmins = [] +) { + $iptables_rule = regsubst ($elasticsearch_masters, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') + class { 'openstack_project::server': + iptables_public_tcp_ports => [22], + iptables_rules6 => $iptables_rule, + iptables_rules4 => $iptables_rule, + sysadmins => $sysadmins, + } + + class { 'logstash::indexer': + conf_template => 'openstack_project/logstash/indexer.conf.erb', + } + + package { 'python3': + ensure => 'present', + } + + package { 'python3-zmq': + ensure => 'present', + } + + package { 'python3-yaml': + ensure => 'present', + } + + file { '/usr/local/bin/log-pusher.py': + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/openstack_project/logstash/log-pusher.py', + require => Package['python3'], + } + + file { '/etc/logstash/jenkins-log-pusher.yaml': + ensure => present, + owner => 'root', + group => 'root', + mode => '0555', + source => "puppet:///modules/openstack_project/logstash/${::hostname}/jenkins-log-pusher.yaml", + require => Class['logstash::indexer'], + } + + file { '/etc/init.d/jenkins-log-pusher': + ensure => present, + owner => 'root', + group => 'root', + mode => '0555', + source => 'puppet:///modules/openstack_project/logstash/jenkins-log-pusher.init', + require => [ + File['/usr/local/bin/log-pusher.py'], + File['/etc/logstash/jenkins-log-pusher.yaml'], + ], + } + + service { 'jenkins-log-pusher': + enable => true, + hasrestart => true, + subscribe => File['/etc/logstash/jenkins-log-pusher.yaml'], + require => File['/etc/init.d/jenkins-log-pusher'], + } +}