Run single node logstash without redis.
logstash.o.o is running into memory contention. The three logstash processes (agent, indexer, web), redis, and elasticsearch are all memory hogs. Remove redis which allows us to consolidate the work of the agent and indexer into a single process. If we decide to add remote logstash agents we will want to add redis to the mix again. This is probably best done on a second host dedicated to redis. Change-Id: I5ba96b4a27d91cb73c9380ed47d0495b9005effa Reviewed-on: https://review.openstack.org/27452 Reviewed-by: Jeremy Stanley <fungi@yuggoth.org> Reviewed-by: James E. Blair <corvus@inaugust.com> Approved: Clark Boylan <clark.boylan@gmail.com> Tested-by: Jenkins
This commit is contained in:
parent
17ac82d873
commit
3a3e0a1f22
|
@ -1,13 +1,46 @@
|
|||
input {
|
||||
redis {
|
||||
host => "127.0.0.1"
|
||||
type => "redis-input"
|
||||
# these settings should match the output of the agent
|
||||
data_type => "list"
|
||||
key => "logstash"
|
||||
pipe {
|
||||
command => "python3 /usr/local/bin/log-pusher.py -r -z tcp://jenkins.openstack.org:8888 -l http://logs.openstack.org -f console.html"
|
||||
format => "json"
|
||||
message_format => "%{event_message}"
|
||||
tags => ["jenkins", "console"]
|
||||
type => "jenkins_console"
|
||||
}
|
||||
}
|
||||
|
||||
# We use json_event here since the sender is a logstash agent
|
||||
format => "json_event"
|
||||
# You can check grok patterns at http://grokdebug.herokuapp.com/
|
||||
filter {
|
||||
grep {
|
||||
type => "jenkins_console"
|
||||
# Drop matches.
|
||||
negate => true
|
||||
match => ["@message", "^</?pre>$"]
|
||||
}
|
||||
multiline {
|
||||
type => "jenkins_console"
|
||||
negate => true
|
||||
pattern => "^%{DATESTAMP} \|"
|
||||
what => "previous"
|
||||
}
|
||||
grok {
|
||||
type => "jenkins_console"
|
||||
pattern => [ "^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
|
||||
add_field => [ "received_at", "%{@timestamp}" ]
|
||||
}
|
||||
date {
|
||||
type => "jenkins_console"
|
||||
exclude_tags => "_grokparsefailure"
|
||||
match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ]
|
||||
}
|
||||
mutate {
|
||||
type => "jenkins_console"
|
||||
exclude_tags => "_grokparsefailure"
|
||||
replace => [ "@message", "%{logmessage}" ]
|
||||
}
|
||||
mutate {
|
||||
type => "jenkins_console"
|
||||
exclude_tags => "_grokparsefailure"
|
||||
remove => [ "logdate", "logmessage" ]
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue