From 7086b8f4599373def2e867a66d1015d8ca065bcd Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Wed, 24 Apr 2013 15:20:14 -0700
Subject: [PATCH] Run single node logstash without redis.

logstash.o.o is running into memory contention. The three logstash
processes (agent, indexer, web), redis, and elasticsearch are all memory
hogs. Remove redis which allows us to consolidate the work of the agent
and indexer into a single process.

If we decide to add remote logstash agents we will want to add redis to
the mix again. This is probably best done on a second host dedicated to
redis.

Change-Id: I5ba96b4a27d91cb73c9380ed47d0495b9005effa
Reviewed-on: https://review.openstack.org/27452
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Reviewed-by: James E. Blair <corvus@inaugust.com>
Approved: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins
---
 .../openstack_project/manifests/logstash.pp   |  8 +--
 .../templates/logstash/indexer.conf.erb       | 49 ++++++++++++++++---
 2 files changed, 45 insertions(+), 12 deletions(-)

diff --git a/modules/openstack_project/manifests/logstash.pp b/modules/openstack_project/manifests/logstash.pp
index 0a0b7947a9..f72676d7be 100644
--- a/modules/openstack_project/manifests/logstash.pp
+++ b/modules/openstack_project/manifests/logstash.pp
@@ -22,16 +22,16 @@ class openstack_project::logstash (
     sysadmins                 => $sysadmins,
   }
 
-  class { 'logstash::agent':
-    conf_template => 'openstack_project/logstash/agent.conf.erb',
-  }
   class { 'logstash::indexer':
     conf_template => 'openstack_project/logstash/indexer.conf.erb',
   }
-  include logstash::redis
   include logstash::elasticsearch
   include logstash::web
 
+  package { 'redis-server':
+    ensure => 'absent',
+  }
+
   package { 'python3':
     ensure => 'present',
   }
diff --git a/modules/openstack_project/templates/logstash/indexer.conf.erb b/modules/openstack_project/templates/logstash/indexer.conf.erb
index 386dbd378d..5ab682591a 100644
--- a/modules/openstack_project/templates/logstash/indexer.conf.erb
+++ b/modules/openstack_project/templates/logstash/indexer.conf.erb
@@ -1,13 +1,46 @@
 input {
-  redis {
-    host => "127.0.0.1"
-    type => "redis-input"
-    # these settings should match the output of the agent
-    data_type => "list"
-    key => "logstash"
+  pipe {
+    command => "python3 /usr/local/bin/log-pusher.py -r -z tcp://jenkins.openstack.org:8888 -l http://logs.openstack.org -f console.html"
+    format => "json"
+    message_format => "%{event_message}"
+    tags => ["jenkins", "console"]
+    type => "jenkins_console"
+  }
+}
 
-    # We use json_event here since the sender is a logstash agent
-    format => "json_event"
+# You can check grok patterns at http://grokdebug.herokuapp.com/
+filter {
+  grep {
+    type => "jenkins_console"
+    # Drop matches.
+    negate => true
+    match => ["@message", "^</?pre>$"]
+  }
+  multiline {
+    type => "jenkins_console"
+    negate => true
+    pattern => "^%{DATESTAMP} \|"
+    what => "previous"
+  }
+  grok {
+    type => "jenkins_console"
+    pattern => [ "^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
+    add_field => [ "received_at", "%{@timestamp}" ]
+  }
+  date {
+    type => "jenkins_console"
+    exclude_tags => "_grokparsefailure"
+    match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ]
+  }
+  mutate {
+    type => "jenkins_console"
+    exclude_tags => "_grokparsefailure"
+    replace => [ "@message", "%{logmessage}" ]
+  }
+  mutate {
+    type => "jenkins_console"
+    exclude_tags => "_grokparsefailure"
+    remove => [ "logdate", "logmessage" ]
   }
 }