Run single node logstash without redis.

logstash.o.o is running into memory contention. The three logstash
processes (agent, indexer, web), redis, and elasticsearch are all memory
hogs. Remove redis which allows us to consolidate the work of the agent
and indexer into a single process.

If we decide to add remote logstash agents we will want to add redis to
the mix again. This is probably best done on a second host dedicated to
redis.

Change-Id: I5ba96b4a27d91cb73c9380ed47d0495b9005effa
Reviewed-on: https://review.openstack.org/27452
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Reviewed-by: James E. Blair <corvus@inaugust.com>
Approved: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins

modules/openstack_project/manifests/logstash.pp

@@ -22,16 +22,16 @@ class openstack_project::logstash (
     sysadmins                 => $sysadmins,
   }
 
-  class { 'logstash::agent':
-    conf_template => 'openstack_project/logstash/agent.conf.erb',
-  }
   class { 'logstash::indexer':
     conf_template => 'openstack_project/logstash/indexer.conf.erb',
   }
-  include logstash::redis
   include logstash::elasticsearch
   include logstash::web
 
+  package { 'redis-server':
+    ensure => 'absent',
+  }
+
   package { 'python3':
     ensure => 'present',
   }

modules/openstack_project/templates/logstash/indexer.conf.erb

@@ -1,13 +1,46 @@
 input {
-  redis {
+  pipe {
-    host => "127.0.0.1"
+    command => "python3 /usr/local/bin/log-pusher.py -r -z tcp://jenkins.openstack.org:8888 -l http://logs.openstack.org -f console.html"
-    type => "redis-input"
+    format => "json"
-    # these settings should match the output of the agent
+    message_format => "%{event_message}"
-    data_type => "list"
+    tags => ["jenkins", "console"]
-    key => "logstash"
+    type => "jenkins_console"
+  }
+}
 
-    # We use json_event here since the sender is a logstash agent
+# You can check grok patterns at http://grokdebug.herokuapp.com/
-    format => "json_event"
+filter {
+  grep {
+    type => "jenkins_console"
+    # Drop matches.
+    negate => true
+    match => ["@message", "^</?pre>$"]
+  }
+  multiline {
+    type => "jenkins_console"
+    negate => true
+    pattern => "^%{DATESTAMP} \|"
+    what => "previous"
+  }
+  grok {
+    type => "jenkins_console"
+    pattern => [ "^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
+    add_field => [ "received_at", "%{@timestamp}" ]
+  }
+  date {
+    type => "jenkins_console"
+    exclude_tags => "_grokparsefailure"
+    match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ]
+  }
+  mutate {
+    type => "jenkins_console"
+    exclude_tags => "_grokparsefailure"
+    replace => [ "@message", "%{logmessage}" ]
+  }
+  mutate {
+    type => "jenkins_console"
+    exclude_tags => "_grokparsefailure"
+    remove => [ "logdate", "logmessage" ]
   }
 }