538793a051
* modules/openstack_project/files/logstash/jenkins-log-pusher.yaml: Add the cinder and quantum screen log files to the log pusher. Tag them with screen and oslofmt (screen because they are screen logs and oslofmt because they use the oslo log message format). * modules/openstack_project/templates/logstash/indexer.conf.erb: Add a grep filter to remove the screen log header lines. Add a multiline filter to handle oslo log format multi line events. Add a grok filter to parse the oslo format logs. Handle timestamps without millisecond precision. Remove event_message field if that message was properly parsed. Change-Id: Icd18e252a512416e0cce5ee0e27942b072a25e09 Reviewed-on: https://review.openstack.org/29985 Reviewed-by: James E. Blair <corvus@inaugust.com> Approved: Clark Boylan <clark.boylan@gmail.com> Reviewed-by: Clark Boylan <clark.boylan@gmail.com> Tested-by: Jenkins
83 lines
2.1 KiB
Plaintext
83 lines
2.1 KiB
Plaintext
input {
|
|
tcp {
|
|
host => "localhost"
|
|
port => 9999
|
|
format => "json"
|
|
message_format => "%{event_message}"
|
|
type => "jenkins"
|
|
}
|
|
}
|
|
|
|
# You can check grok patterns at http://grokdebug.herokuapp.com/
|
|
filter {
|
|
grep {
|
|
# Remove unneeded html tags.
|
|
type => "jenkins"
|
|
tags => ["console.html"]
|
|
# Drop matches.
|
|
negate => true
|
|
match => ["@message", "^</?pre>$"]
|
|
}
|
|
grep {
|
|
# Remove screen log headers.
|
|
type => "jenkins"
|
|
tags => ["screen"]
|
|
# Drop matches.
|
|
negate => true
|
|
match => ["@message", "^\+ "]
|
|
}
|
|
multiline {
|
|
type => "jenkins"
|
|
tags => ["console.html"]
|
|
negate => true
|
|
pattern => "^%{DATESTAMP} \|"
|
|
what => "previous"
|
|
}
|
|
multiline {
|
|
type => "jenkins"
|
|
tags => ["oslofmt"]
|
|
negate => true
|
|
pattern => "^%{DATESTAMP} "
|
|
what => "previous"
|
|
}
|
|
grok {
|
|
type => "jenkins"
|
|
tags => ["console.html"]
|
|
# Do multiline matching as the above mutliline filter may add newlines
|
|
# to the log messages.
|
|
pattern => [ "(?m)^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
|
|
add_field => [ "received_at", "%{@timestamp}" ]
|
|
}
|
|
grok {
|
|
type => "jenkins"
|
|
tags => ["oslofmt"]
|
|
# Do multiline matching as the above mutliline filter may add newlines
|
|
# to the log messages.
|
|
# TODO move the LOGLEVELs into a proper grok pattern.
|
|
pattern => [ "(?m)^%{DATESTAMP:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" ]
|
|
add_field => [ "received_at", "%{@timestamp}" ]
|
|
}
|
|
date {
|
|
type => "jenkins"
|
|
exclude_tags => "_grokparsefailure"
|
|
match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss" ]
|
|
}
|
|
mutate {
|
|
type => "jenkins"
|
|
exclude_tags => "_grokparsefailure"
|
|
replace => [ "@message", "%{logmessage}" ]
|
|
}
|
|
mutate {
|
|
type => "jenkins"
|
|
exclude_tags => "_grokparsefailure"
|
|
remove => [ "logdate", "logmessage", "event_message" ]
|
|
}
|
|
}
|
|
|
|
output {
|
|
elasticsearch {
|
|
host => "elasticsearch.openstack.org"
|
|
node_name => "logstash-indexer"
|
|
}
|
|
}
|