From 219cef8d065a39312a6b3755222a7bb37e7f863a Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Tue, 6 Aug 2013 11:52:26 -0700
Subject: [PATCH] Don't index logs with DEBUG log level.

ElasticSearch has a hard time performing queries on large amounts of
data. It must load the fields it is searching on into memory and that
can cause the node to use all of the memory allocated to it which in
turn causes a lot of garbage collection essentially taking the node
offline. Filter out DEBUG log messages in the devstack logs to reduce
the amount of data that must be loaded into memory when performing
searches.

Change-Id: Icfe9c8c17ccef4f9379c774eef791f43463dcf6d
---
 .../openstack_project/templates/logstash/indexer.conf.erb   | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/modules/openstack_project/templates/logstash/indexer.conf.erb b/modules/openstack_project/templates/logstash/indexer.conf.erb
index d55505923b..fec7b5d485 100644
--- a/modules/openstack_project/templates/logstash/indexer.conf.erb
+++ b/modules/openstack_project/templates/logstash/indexer.conf.erb
@@ -106,6 +106,12 @@ filter {
     pattern => [ "%{SYSLOGTIMESTAMP:logdate}%{SPACE}%{SYSLOGHOST:syslog_host}?%{SPACE}%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:? %{GREEDYDATA:logmessage}" ]
     add_field => [ "received_at", "%{@timestamp}" ]
   }
+  # Remove DEBUG logs to reduce the amount of data that needs to be processed.
+  grep {
+    type => "jenkins"
+    negate => true
+    match => [ "loglevel", "DEBUG" ]
+  }
 
   # Filters below here should be consistent for all Jenkins log formats.
   date {