Scale out logstash indexing to multiple hosts.

Logstash performs filtering in a single thread so it does not scale up
very well. Work around this by scaling Logstash out to multiple indexer
hosts.

Current plan is to have a small (2GB) kibana web front end host that
does nothing but talk to elasticsearch, three 4GB logstash indexers that
will run a single log-pusher.py + logstash indexer with some partition
of the logfiles assigned to each indexer, and finally the existing large
elasticsearch node.

Eventually properly load balancing log processing across the worker
nodes would be great, but the current partition method should work well
enough with little additional effort.

Change-Id: Ifc6396560934314ffd6a7c47eb2acff9e9c2a7af
Reviewed-on: https://review.openstack.org/30573
Reviewed-by: James E. Blair <corvus@inaugust.com>
Approved: Jeremy Stanley <fungi@yuggoth.org>
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Tested-by: Jenkins

manifests/site.pp

@@ -75,6 +75,11 @@ node 'jenkins.openstack.org' {
     ssl_key_file_contents   => hiera('jenkins_ssl_key_file_contents'),
     ssl_chain_file_contents => hiera('jenkins_ssl_chain_file_contents'),
     sysadmins               => hiera('sysadmins'),
+    zmq_event_receivers     => [
+      'logstash-worker1.openstack.org',
+      'logstash-worker2.openstack.org',
+      'logstash-worker3.openstack.org',
+    ],
   }
 }
 
@@ -200,13 +205,27 @@ node 'puppet-dashboard.openstack.org' {
 
 node 'logstash.openstack.org' {
   class { 'openstack_project::logstash':
-    sysadmins => hiera('sysadmins'),
+    sysadmins             => hiera('sysadmins'),
+    elasticsearch_masters => ['elasticsearch.openstack.org'],
+  }
+}
+
+node /^logstash-worker\d+\.openstack\.org$/ {
+  class { 'openstack_project::logstash_worker':
+    sysadmins             => hiera('sysadmins'),
+    elasticsearch_masters => ['elasticsearch.openstack.org'],
   }
 }
 
 node 'elasticsearch.openstack.org' {
   class { 'openstack_project::elasticsearch':
-    sysadmins => hiera('sysadmins'),
+    sysadmins        => hiera('sysadmins'),
+    logstash_workers => [
+      'logstash.openstack.org',
+      'logstash-worker1.openstack.org',
+      'logstash-worker2.openstack.org',
+      'logstash-worker3.openstack.org',
+    ],
   }
 }
 

modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml

@@ -0,0 +1,29 @@
+# Defaults
+source-defaults:
+  source-url: http://logs.openstack.org
+  output-host: localhost
+  output-port: 9999
+  output-mode: tcp
+  retry-get: False
+
+# List of zmq event inputs.
+zmq-publishers:
+  - tcp://jenkins.openstack.org:8888
+
+# List of files to source logs from.
+source-files:
+  - name: console.html
+    retry-get: True
+  - name: logs/screen-c-api.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-c-sch.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-c-vol.txt
+    tags:
+      - screen
+      - oslofmt
+# TODO(clarkb) Add keystone, swift, and syslog logs here.

modules/openstack_project/files/logstash/logstash-worker2/jenkins-log-pusher.yaml

@@ -0,0 +1,42 @@
+# Defaults
+source-defaults:
+  source-url: http://logs.openstack.org
+  output-host: localhost
+  output-port: 9999
+  output-mode: tcp
+  retry-get: False
+
+# List of zmq event inputs.
+zmq-publishers:
+  - tcp://jenkins.openstack.org:8888
+
+# List of files to source logs from.
+source-files:
+  - name: logs/screen-n-api.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-n-cond.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-n-cpu.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-n-crt.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-n-net.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-n-obj.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-n-sch.txt
+    tags:
+      - screen
+      - oslofmt

modules/openstack_project/files/logstash/logstash-worker3/jenkins-log-pusher.yaml

@@ -12,20 +12,6 @@ zmq-publishers:
 
 # List of files to source logs from.
 source-files:
-  - name: console.html
-    retry-get: True
-  - name: logs/screen-c-api.txt
-    tags:
-      - screen
-      - oslofmt
-  - name: logs/screen-c-sch.txt
-    tags:
-      - screen
-      - oslofmt
-  - name: logs/screen-c-vol.txt
-    tags:
-      - screen
-      - oslofmt
   - name: logs/screen-g-api.txt
     tags:
       - screen
@@ -34,30 +20,6 @@ source-files:
     tags:
       - screen
       - oslofmt
-  - name: logs/screen-n-api.txt
-    tags:
-      - screen
-      - oslofmt
-  - name: logs/screen-n-cond.txt
-    tags:
-      - screen
-      - oslofmt
-  - name: logs/screen-n-cpu.txt
-    tags:
-      - screen
-      - oslofmt
-  - name: logs/screen-n-crt.txt
-    tags:
-      - screen
-      - oslofmt
-  - name: logs/screen-n-obj.txt
-    tags:
-      - screen
-      - oslofmt
-  - name: logs/screen-n-sch.txt
-    tags:
-      - screen
-      - oslofmt
   - name: logs/screen-q-agt.txt
     tags:
       - screen

modules/openstack_project/manifests/elasticsearch.pp

@@ -15,9 +15,10 @@
 # Elasticsearch server glue class.
 #
 class openstack_project::elasticsearch (
+  $logstash_workers = [],
   $sysadmins = []
 ) {
-  $iptables_rule = '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s logstash.openstack.org -j ACCEPT'
+  $iptables_rule = regsubst ($logstash_workers, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT')
   class { 'openstack_project::server':
     iptables_public_tcp_ports => [22],
     iptables_rules6           => $iptables_rule,

modules/openstack_project/manifests/jenkins.pp

@@ -7,11 +7,12 @@ class openstack_project::jenkins (
   $ssl_key_file_contents = '',
   $ssl_chain_file_contents = '',
   $jenkins_ssh_private_key = '',
+  $zmq_event_receivers = [],
   $sysadmins = []
 ) {
   include openstack_project
 
-  $iptables_rule = '-m state --state NEW -m tcp -p tcp --dport 8888 -s logstash.openstack.org -j ACCEPT'
+  $iptables_rule = regsubst ($zmq_event_receivers, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 8888 -s \1 -j ACCEPT')
   class { 'openstack_project::server':
     iptables_public_tcp_ports => [80, 443],
     iptables_rules6           => $iptables_rule,

modules/openstack_project/manifests/logstash.pp

@@ -12,12 +12,13 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 #
-# Logstash indexer server glue class.
+# Logstash web frontend glue class.
 #
 class openstack_project::logstash (
+  $elasticsearch_masters = [],
   $sysadmins = []
 ) {
-  $iptables_rule = '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s elasticsearch.openstack.org -j ACCEPT'
+  $iptables_rule = regsubst ($elasticsearch_masters, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT')
   class { 'openstack_project::server':
     iptables_public_tcp_ports => [22, 80],
     iptables_rules6           => $iptables_rule,
@@ -25,60 +26,8 @@ class openstack_project::logstash (
     sysadmins                 => $sysadmins,
   }
 
-  class { 'logstash::indexer':
-    conf_template => 'openstack_project/logstash/indexer.conf.erb',
-  }
   class { 'logstash::web':
     frontend           => 'kibana',
     elasticsearch_host => 'elasticsearch.openstack.org',
   }
-
-  package { 'python3':
-    ensure => 'present',
-  }
-
-  package { 'python3-zmq':
-    ensure => 'present',
-  }
-
-  package { 'python3-yaml':
-    ensure => 'present',
-  }
-
-  file { '/usr/local/bin/log-pusher.py':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0755',
-    source  => 'puppet:///modules/openstack_project/logstash/log-pusher.py',
-    require => Package['python3'],
-  }
-
-  file { '/etc/logstash/jenkins-log-pusher.yaml':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0555',
-    source  => 'puppet:///modules/openstack_project/logstash/jenkins-log-pusher.yaml',
-    require => Class['logstash::indexer'],
-  }
-
-  file { '/etc/init.d/jenkins-log-pusher':
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0555',
-    source  => 'puppet:///modules/openstack_project/logstash/jenkins-log-pusher.init',
-    require => [
-      File['/usr/local/bin/log-pusher.py'],
-      File['/etc/logstash/jenkins-log-pusher.yaml'],
-    ],
-  }
-
-  service { 'jenkins-log-pusher':
-    enable     => true,
-    hasrestart => true,
-    subscribe  => File['/etc/logstash/jenkins-log-pusher.yaml'],
-    require    => File['/etc/init.d/jenkins-log-pusher'],
-  }
 }

modules/openstack_project/manifests/logstash_worker.pp

@@ -0,0 +1,81 @@
+# Copyright 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Logstash indexer worker glue class.
+#
+class openstack_project::logstash_worker (
+  $elasticsearch_masters = [],
+  $sysadmins = []
+) {
+  $iptables_rule = regsubst ($elasticsearch_masters, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT')
+  class { 'openstack_project::server':
+    iptables_public_tcp_ports => [22],
+    iptables_rules6           => $iptables_rule,
+    iptables_rules4           => $iptables_rule,
+    sysadmins                 => $sysadmins,
+  }
+
+  class { 'logstash::indexer':
+    conf_template => 'openstack_project/logstash/indexer.conf.erb',
+  }
+
+  package { 'python3':
+    ensure => 'present',
+  }
+
+  package { 'python3-zmq':
+    ensure => 'present',
+  }
+
+  package { 'python3-yaml':
+    ensure => 'present',
+  }
+
+  file { '/usr/local/bin/log-pusher.py':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    source  => 'puppet:///modules/openstack_project/logstash/log-pusher.py',
+    require => Package['python3'],
+  }
+
+  file { '/etc/logstash/jenkins-log-pusher.yaml':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0555',
+    source  => "puppet:///modules/openstack_project/logstash/${::hostname}/jenkins-log-pusher.yaml",
+    require => Class['logstash::indexer'],
+  }
+
+  file { '/etc/init.d/jenkins-log-pusher':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0555',
+    source  => 'puppet:///modules/openstack_project/logstash/jenkins-log-pusher.init',
+    require => [
+      File['/usr/local/bin/log-pusher.py'],
+      File['/etc/logstash/jenkins-log-pusher.yaml'],
+    ],
+  }
+
+  service { 'jenkins-log-pusher':
+    enable     => true,
+    hasrestart => true,
+    subscribe  => File['/etc/logstash/jenkins-log-pusher.yaml'],
+    require    => File['/etc/init.d/jenkins-log-pusher'],
+  }
+}