Use log-pusher injected tags during logstash grok

* modules/openstack_project/templates/logstash/indexer.conf.erb: The
log-pusher.py script tags events with the filename of the log generating
the event. Use these values instead of different types to differentiate
Jenkins console logs from nova logs and so on.

Note that filters must match all of the values in the tags array (they
are ANDed together). This may mean that as the logstash filter rulesets
grow we will need a set of initial mutates to convert
'logs/screen-n-api.txt' and 'logs/screen-n-cpu.txt' tags to 'nova' to
allow for common grok and parsing filters.

Change-Id: I2769bc05a2e9cc7e8dbc46849e052146b9fee75e
Reviewed-on: https://review.openstack.org/28119
Reviewed-by: James E. Blair <corvus@inaugust.com>
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: Jeremy Stanley <fungi@yuggoth.org>
Tested-by: Jenkins
This commit is contained in:
Clark Boylan 2013-05-02 16:02:19 -07:00 committed by Jenkins
parent aa411bef1a
commit 929f152d49
1 changed files with 10 additions and 8 deletions

View File

@ -4,42 +4,44 @@ input {
port => 9999
format => "json"
message_format => "%{event_message}"
tags => ["jenkins", "console"]
type => "jenkins_console"
type => "jenkins"
}
}
# You can check grok patterns at http://grokdebug.herokuapp.com/
filter {
grep {
type => "jenkins_console"
type => "jenkins"
tags => ["console.html"]
# Drop matches.
negate => true
match => ["@message", "^</?pre>$"]
}
multiline {
type => "jenkins_console"
type => "jenkins"
tags => ["console.html"]
negate => true
pattern => "^%{DATESTAMP} \|"
what => "previous"
}
grok {
type => "jenkins_console"
type => "jenkins"
tags => ["console.html"]
pattern => [ "^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
add_field => [ "received_at", "%{@timestamp}" ]
}
date {
type => "jenkins_console"
type => "jenkins"
exclude_tags => "_grokparsefailure"
match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ]
}
mutate {
type => "jenkins_console"
type => "jenkins"
exclude_tags => "_grokparsefailure"
replace => [ "@message", "%{logmessage}" ]
}
mutate {
type => "jenkins_console"
type => "jenkins"
exclude_tags => "_grokparsefailure"
remove => [ "logdate", "logmessage" ]
}