206 lines
6.5 KiB
Plaintext
206 lines
6.5 KiB
Plaintext
# logstash.conf
|
|
#
|
|
# Basic logstash config and filters for injesting most logs from Nova,
|
|
# Keystone, Glance, Ceilometer, Heat, and Neutron services, as well as
|
|
# Apache (Horizon) and syslog.
|
|
#
|
|
# Author: Kris Lindgren <klindgren@godaddy.com>
|
|
#
|
|
# Copyright (c) 2014 Go Daddy Operating Company, LLC
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a
|
|
# copy of this software and associated documentation files (the "Software"),
|
|
# to deal in the Software without restriction, including without limitation
|
|
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
# and/or sell copies of the Software, and to permit persons to whom the
|
|
# Software is furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in
|
|
# all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
# DEALINGS IN THE SOFTWARE.
|
|
#
|
|
|
|
input {
|
|
file {
|
|
path => ['/var/log/nova/nova-api.log']
|
|
tags => ['nova', 'oslofmt']
|
|
type => "nova"
|
|
}
|
|
file {
|
|
path => ['/var/log/nova/nova-conductor.log']
|
|
tags => ['nova', 'oslofmt']
|
|
type => "nova"
|
|
}
|
|
file {
|
|
path => ['/var/log/nova/nova-manage.log']
|
|
tags => ['nova', 'oslofmt']
|
|
type => "nova"
|
|
}
|
|
file {
|
|
path => ['/var/log/nova/nova-scheduler.log']
|
|
tags => ['nova', 'oslofmt']
|
|
type => "nova"
|
|
}
|
|
file {
|
|
path => ['/var/log/nova/nova-spicehtml5proxy.log']
|
|
tags => ['nova', 'oslofmt']
|
|
type => "nova"
|
|
}
|
|
file {
|
|
path => ['/var/log/keystone/keystone-all.log']
|
|
tags => ['keystone', 'keystonefmt']
|
|
type => "keystone"
|
|
}
|
|
file {
|
|
path => ['/var/log/keystone/keystone-manage.log']
|
|
tags => ['keystone', 'keystonefmt']
|
|
type => "keystone"
|
|
}
|
|
file {
|
|
path => ['/var/log/glance/api.log']
|
|
tags => ['glance', 'oslofmt']
|
|
type => "glance"
|
|
}
|
|
file {
|
|
path => ['/var/log/glance/registry.log']
|
|
tags => ['glance', 'oslofmt']
|
|
type => "glance"
|
|
}
|
|
file {
|
|
path => ['/var/log/glance/scrubber.log']
|
|
tags => ['glance', 'oslofmt']
|
|
type => "glance"
|
|
}
|
|
file {
|
|
path => ['/var/log/ceilometer/ceilometer-agent-central.log']
|
|
tags => ['ceilometer', 'oslofmt']
|
|
type => "ceilometer"
|
|
}
|
|
file {
|
|
path => ['/var/log/ceilometer/ceilometer-alarm-notifier.log']
|
|
tags => ['ceilometer', 'oslofmt']
|
|
type => "ceilometer"
|
|
}
|
|
file {
|
|
path => ['/var/log/ceilometer/ceilometer-api.log']
|
|
tags => ['ceilometer', 'oslofmt']
|
|
type => "ceilometer"
|
|
}
|
|
file {
|
|
path => ['/var/log/ceilometer/ceilometer-alarm-evaluator.log']
|
|
tags => ['ceilometer', 'oslofmt']
|
|
type => "ceilometer"
|
|
}
|
|
file {
|
|
path => ['/var/log/ceilometer/ceilometer-collector.log']
|
|
tags => ['ceilometer', 'oslofmt']
|
|
type => "ceilometer"
|
|
}
|
|
file {
|
|
path => ['/var/log/heat/heat.log']
|
|
tags => ['heat', 'oslofmt']
|
|
type => "heat"
|
|
}
|
|
file {
|
|
path => ['/var/log/neutron/neutron-server.log']
|
|
tags => ['neutron', 'oslofmt']
|
|
type => "neutron"
|
|
}
|
|
# Not collecting RabbitMQ logs for the moment
|
|
# file {
|
|
# path => ['/var/log/rabbitmq/rabbit@<%= @hostname %>.log']
|
|
# tags => ['rabbitmq', 'oslofmt']
|
|
# type => "rabbitmq"
|
|
# }
|
|
file {
|
|
path => ['/var/log/httpd/access_log']
|
|
tags => ['horizon']
|
|
type => "horizon"
|
|
}
|
|
file {
|
|
path => ['/var/log/httpd/error_log']
|
|
tags => ['horizon']
|
|
type => "horizon"
|
|
}
|
|
file {
|
|
path => ['/var/log/httpd/horizon_access_log']
|
|
tags => ['horizon']
|
|
type => "horizon"
|
|
}
|
|
file {
|
|
path => ['/var/log/httpd/horizon_error_log']
|
|
tags => ['horizon']
|
|
type => "horizon"
|
|
}
|
|
}
|
|
filter {
|
|
if "oslofmt" in [tags] {
|
|
multiline {
|
|
negate => true
|
|
pattern => "^%{TIMESTAMP_ISO8601} "
|
|
what => "previous"
|
|
}
|
|
multiline {
|
|
negate => false
|
|
pattern => "^%{TIMESTAMP_ISO8601}%{SPACE}%{NUMBER}?%{SPACE}?TRACE"
|
|
what => "previous"
|
|
}
|
|
grok {
|
|
# Do multiline matching as the above mutliline filter may add newlines
|
|
# to the log messages.
|
|
# TODO move the LOGLEVELs into a proper grok pattern.
|
|
match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
|
|
add_field => { "received_at" => "%{@timestamp}" }
|
|
}
|
|
|
|
} else if "keystonefmt" in [tags] {
|
|
grok {
|
|
# Do multiline matching as the above mutliline filter may add newlines
|
|
# to the log messages.
|
|
# TODO move the LOGLEVELs into a proper grok pattern.
|
|
match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
|
|
add_field => { "received_at" => "%{@timestamp}" }
|
|
}
|
|
if [module] == "iso8601.iso8601" {
|
|
#log message for each part of the date? Really?
|
|
drop {}
|
|
}
|
|
} else if "libvirt" in [tags] {
|
|
grok {
|
|
match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}:%{SPACE}%{NUMBER:code}:?%{SPACE}\[?\b%{NOTSPACE:loglevel}\b\]?%{SPACE}?:?%{SPACE}\[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
|
|
add_field => { "received_at" => "%{@timestamp}"}
|
|
}
|
|
mutate {
|
|
uppercase => [ "loglevel" ]
|
|
}
|
|
} else if [type] == "syslog" {
|
|
grok {
|
|
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:logmessage}" }
|
|
add_field => [ "received_at", "%{@timestamp}" ]
|
|
}
|
|
syslog_pri {
|
|
severity_labels => ["ERROR", "ERROR", "ERROR", "ERROR", "WARNING", "INFO", "INFO", "DEBUG" ]
|
|
}
|
|
date {
|
|
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
|
|
}
|
|
if !("_grokparsefailure" in [tags]) {
|
|
mutate {
|
|
replace => [ "@source_host", "%{syslog_hostname}" ]
|
|
}
|
|
}
|
|
mutate {
|
|
remove_field => [ "syslog_hostname", "syslog_timestamp" ]
|
|
add_field => [ "loglevel", "%{syslog_severity}" ]
|
|
add_field => [ "module", "%{syslog_program}" ]
|
|
}
|
|
}
|
|
}
|