diff --git a/logstash/basic/logstash.conf b/logstash/basic/logstash.conf index 3223e55..d86dadf 100644 --- a/logstash/basic/logstash.conf +++ b/logstash/basic/logstash.conf @@ -159,18 +159,71 @@ filter { match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" } add_field => { "received_at" => "%{@timestamp}" } } - - } else if "keystonefmt" in [tags] { - grok { - # Do multiline matching as the above mutliline filter may add newlines - # to the log messages. - # TODO move the LOGLEVELs into a proper grok pattern. - match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" } - add_field => { "received_at" => "%{@timestamp}" } - } if [module] == "iso8601.iso8601" { - #log message for each part of the date? Really? - drop {} + drop {} + } + + if "keystoneapi" in [tags] { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[\-\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:method} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes} %{NUMBER:seconds}" } + add_field => ["api", "keystone"] + add_tag => ["apimetrics"] + } + } else if "novaapi" in [tags] { + if [module] == "nova.osapi_compute.wsgi.server" { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[req\-%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant}\] %{NOTSPACE:requesterip} %{NOTSPACE:method} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes} time\: %{NUMBER:seconds}" } + add_field => ["api", "nova"] + add_tag => ["apimetrics"] + } + } + } else if "neutronapi" in [tags] { + if [module] == "neutron.wsgi" { + if "accepted" not in [logmessage] { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[req\-%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant}\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:method} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes} %{NUMBER:seconds}" } + add_field => ["api", "neutron"] + add_tag => ["apimetrics"] + } + } + } + } else if "glanceapi" in [tags] { + if [module] == "glance.wsgi.server" { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:method} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes} %{NUMBER:seconds}" } + add_field => ["api", "glance"] + add_tag => ["apimetrics"] + } + } + } else if "novametaapi" in [tags] { + mutate { + gsub => ['logmessage',"\"",""] + } + if [module] == "nova.api.ec2" { + grok { + match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NUMBER:seconds}s %{NOTSPACE:requesterip} %{NOTSPACE:method} %{NOTSPACE:url_path} None\:None %{NUMBER:response} %{GREEDYDATA:user_agent}" } + add_field => ["api", "metadata-ec2"] + add_tag => ["apimetrics"] + } + } else if [module] == "nova.metadata.wsgi.server" { + grok { + match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NOTSPACE:requesterip} %{NOTSPACE:method} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes} time\: %{NUMBER:seconds}" } + add_field => ["api", "metadata"] + add_tag => ["apimetrics"] + } + } } } else if "libvirt" in [tags] { grok {