Remove no longer used logstash config
This created confusion when updating configs to handle journald. Remove the unused files and update docs to point at the proper config location. Change-Id: Ifd8d8868b124b72a86cf7b5acb30480e72b903ed
This commit is contained in:
Clark Boylan
parent
4436901673
commit
90e867b0e3
|
|
@ -172,23 +172,23 @@ schema.
|
||||||
|
|
||||||
The config file that tells Logstash how to do this flattening can be
|
The config file that tells Logstash how to do this flattening can be
|
||||||
found at
|
found at
|
||||||
:cgit_file:`modules/openstack_project/templates/logstash/indexer.conf.erb`
|
https://git.openstack.org/cgit/openstack-infra/logstash-filters/tree/filters/openstack-filters.conf
|
||||||
|
|
||||||
This works via the tags that are associated with a given message.
|
This works via the tags that are associated with a given message.
|
||||||
|
|
||||||
The tags in
|
The tags in
|
||||||
:cgit_file:`modules/openstack_project/templates/logstash/indexer.conf.erb`
|
https://git.openstack.org/cgit/openstack-infra/logstash-filters/tree/filters/openstack-filters.conf
|
||||||
are used to tell logstash how to parse a given file's messages, based
|
are used to tell logstash how to parse a given file's messages, based
|
||||||
on the file's message format.
|
on the file's message format.
|
||||||
|
|
||||||
When adding a new file to be indexed to
|
When adding a new file to be indexed to
|
||||||
http://git.openstack.org/cgit/openstack-infra/project-config/tree/roles/submit-logstash-jobs/defaults/main.yaml
|
http://git.openstack.org/cgit/openstack-infra/project-config/tree/roles/submit-logstash-jobs/defaults/main.yaml
|
||||||
at least one tag from the indexer.conf.erb file should be associated
|
at least one tag from the openstack-filters.conf file should be associated
|
||||||
with the new file. One can expect to see '{%logmessage%}' instead of
|
with the new file. One can expect to see '{%logmessage%}' instead of
|
||||||
actual message data if indexing is not working properly.
|
actual message data if indexing is not working properly.
|
||||||
|
|
||||||
In the event a new file's format is not covered, a patch for
|
In the event a new file's format is not covered, a patch for
|
||||||
:cgit_file:`modules/openstack_project/templates/logstash/indexer.conf.erb`
|
https://git.openstack.org/cgit/openstack-infra/logstash-filters/tree/filters/openstack-filters.conf
|
||||||
should be submitted with an appropriate parsing pattern.
|
should be submitted with an appropriate parsing pattern.
|
||||||
|
|
||||||
ElasticSearch
|
ElasticSearch
|
||||||
|
|
|
||||||
|
|
@ -1,47 +0,0 @@
|
||||||
input {
|
|
||||||
pipe {
|
|
||||||
command => "python3 /usr/local/bin/log-pusher.py -r -z tcp://jenkins.openstack.org:8888 -l http://logs.openstack.org -f console.html"
|
|
||||||
format => "json"
|
|
||||||
message_format => "%{event_message}"
|
|
||||||
tags => ["jenkins", "console"]
|
|
||||||
type => "jenkins_console"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# You can check grok patterns at http://grokdebug.herokuapp.com/
|
|
||||||
filter {
|
|
||||||
grep {
|
|
||||||
type => "jenkins_console"
|
|
||||||
# Drop matches.
|
|
||||||
negate => true
|
|
||||||
match => ["@message", "^</?pre>$"]
|
|
||||||
}
|
|
||||||
multiline {
|
|
||||||
type => "jenkins_console"
|
|
||||||
negate => true
|
|
||||||
pattern => "^%{DATESTAMP} \|"
|
|
||||||
what => "previous"
|
|
||||||
}
|
|
||||||
grok {
|
|
||||||
type => "jenkins_console"
|
|
||||||
pattern => [ "^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
|
|
||||||
add_field => [ "received_at", "%{@timestamp}" ]
|
|
||||||
}
|
|
||||||
date {
|
|
||||||
type => "jenkins_console"
|
|
||||||
exclude_tags => "_grokparsefailure"
|
|
||||||
match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ]
|
|
||||||
}
|
|
||||||
mutate {
|
|
||||||
type => "jenkins_console"
|
|
||||||
exclude_tags => "_grokparsefailure"
|
|
||||||
replace => [ "@message", "%{logmessage}" ]
|
|
||||||
}
|
|
||||||
mutate {
|
|
||||||
type => "jenkins_console"
|
|
||||||
exclude_tags => "_grokparsefailure"
|
|
||||||
remove => [ "logdate", "logmessage" ]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
<%= scope.function_template(['openstack_project/logstash/redis-output.conf.erb']) %>
|
|
||||||
|
|
@ -1,124 +0,0 @@
|
||||||
input {
|
|
||||||
tcp {
|
|
||||||
host => "localhost"
|
|
||||||
port => 9999
|
|
||||||
codec => json_lines {}
|
|
||||||
type => "jenkins"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# You can check grok patterns at http://grokdebug.herokuapp.com/
|
|
||||||
filter {
|
|
||||||
if "screen" in [tags] and [message] =~ "^\+ " {
|
|
||||||
drop {}
|
|
||||||
}
|
|
||||||
if "console" in [tags] or "console.html" in [tags] {
|
|
||||||
if [message] == "<pre>" or [message] == "</pre>" {
|
|
||||||
drop {}
|
|
||||||
}
|
|
||||||
multiline {
|
|
||||||
negate => true
|
|
||||||
pattern => "^%{TIMESTAMP_ISO8601} \|"
|
|
||||||
what => "previous"
|
|
||||||
stream_identity => "%{host}.%{filename}"
|
|
||||||
}
|
|
||||||
grok {
|
|
||||||
# Do multiline matching as the above mutliline filter may add newlines
|
|
||||||
# to the log messages.
|
|
||||||
match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate} \| %{GREEDYDATA:logmessage}" }
|
|
||||||
add_field => { "received_at" => "%{@timestamp}" }
|
|
||||||
}
|
|
||||||
} else if "oslofmt" in [tags] {
|
|
||||||
multiline {
|
|
||||||
negate => true
|
|
||||||
pattern => "^(%{TIMESTAMP_ISO8601}|%{SYSLOGTIMESTAMP}) "
|
|
||||||
what => "previous"
|
|
||||||
stream_identity => "%{host}.%{filename}"
|
|
||||||
}
|
|
||||||
multiline {
|
|
||||||
negate => false
|
|
||||||
# NOTE(mriedem): oslo.log 1.2.0 changed the logging_exception_prefix
|
|
||||||
# config option from using TRACE to ERROR so we have to handle both.
|
|
||||||
#
|
|
||||||
# NOTE(sdague): stack traces always include process id, so
|
|
||||||
# NUMBER being required element here is important, otherwise
|
|
||||||
# ERROR messages just fold into the previous messages, which are
|
|
||||||
# typically INFO.
|
|
||||||
pattern => "^(%{TIMESTAMP_ISO8601}|%{SYSLOGTIMESTAMP})%{SPACE}%{NUMBER}%{SPACE}(TRACE|ERROR)"
|
|
||||||
what => "previous"
|
|
||||||
stream_identity => "%{host}.%{filename}"
|
|
||||||
}
|
|
||||||
grok {
|
|
||||||
# Do multiline matching as the above mutliline filter may add newlines
|
|
||||||
# to the log messages.
|
|
||||||
# TODO move the LOGLEVELs into a proper grok pattern.
|
|
||||||
match => { "message" => "(?m)^(%{TIMESTAMP_ISO8601:logdate}|%{SYSLOGTIMESTAMP:logdate})%{SPACE}(%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:|%{NUMBER:pid})?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
|
|
||||||
add_field => { "received_at" => "%{@timestamp}" }
|
|
||||||
}
|
|
||||||
} else if "apachecombined" in [tags] {
|
|
||||||
grok {
|
|
||||||
match => { "message" => "%{COMBINEDAPACHELOG}" }
|
|
||||||
add_field => { "received_at" => "%{@timestamp}" }
|
|
||||||
add_field => { "logdate" => "%{timestamp}" }
|
|
||||||
add_field => { "logmessage" => "%{verb} %{request} %{response}" }
|
|
||||||
}
|
|
||||||
} else if "apacheerror" in [tags] {
|
|
||||||
grok {
|
|
||||||
match => { "message" => "\[(?<logdate>%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}%{SPACE}%{TZ}?)\]%{SPACE}\[%{LOGLEVEL:loglevel}\]%{SPACE}%{GREEDYDATA:logmessage}" }
|
|
||||||
add_field => { "received_at" => "%{@timestamp}" }
|
|
||||||
}
|
|
||||||
} else if "libvirt" in [tags] {
|
|
||||||
grok {
|
|
||||||
# libvirtd grok filter adapted from
|
|
||||||
# https://github.com/OpenStratus/openstack-logstash/blob/master/agent.conf
|
|
||||||
match => { "message" => "%{TIMESTAMP_ISO8601:logdate}:%{SPACE}%{NUMBER:pid}:%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}:%{SPACE}%{GREEDYDATA:logmessage}" }
|
|
||||||
add_field => { "received_at" => "%{@timestamp}" }
|
|
||||||
}
|
|
||||||
} else if "syslog" in [tags] {
|
|
||||||
grok {
|
|
||||||
# Syslog grok filter adapted from
|
|
||||||
# http://cookbook.logstash.net/recipes/syslog-pri/syslog.conf
|
|
||||||
match => { "message" => "%{SYSLOGTIMESTAMP:logdate}%{SPACE}%{SYSLOGHOST:syslog_host}?%{SPACE}%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:? %{GREEDYDATA:logmessage}" }
|
|
||||||
add_field => { "received_at" => "%{@timestamp}" }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Filters below here should be consistent for all Jenkins log formats.
|
|
||||||
# Remove DEBUG logs to reduce the amount of data that needs to be processed.
|
|
||||||
if [loglevel] == "DEBUG" {
|
|
||||||
drop {}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ! ("_grokparsefailure" in [tags]) {
|
|
||||||
date {
|
|
||||||
match => [ "logdate",
|
|
||||||
"yyyy-MM-dd HH:mm:ss.SSS",
|
|
||||||
"yyyy-MM-dd HH:mm:ss.SSSSSS",
|
|
||||||
"yyyy-MM-dd HH:mm:ss,SSS",
|
|
||||||
"yyyy-MM-dd HH:mm:ss",
|
|
||||||
"MMM d HH:mm:ss",
|
|
||||||
"MMM dd HH:mm:ss",
|
|
||||||
"dd/MMM/yyyy:HH:mm:ss Z",
|
|
||||||
"yyyy-MM-dd HH:mm:ss.SSSZ",
|
|
||||||
"E MMM dd HH:mm:ss yyyy Z",
|
|
||||||
"E MMM dd HH:mm:ss yyyy",
|
|
||||||
"ISO8601"
|
|
||||||
]
|
|
||||||
timezone => "UTC"
|
|
||||||
}
|
|
||||||
mutate {
|
|
||||||
replace => { "message" => "%{logmessage}" }
|
|
||||||
}
|
|
||||||
mutate {
|
|
||||||
remove_field => [ "logdate", "logmessage" ]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
output {
|
|
||||||
elasticsearch {
|
|
||||||
hosts => <%= @elasticsearch_nodes.map { |node| node + ":9200" }.inspect %>
|
|
||||||
manage_template => false
|
|
||||||
flush_size => 1024
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue