Browse Source

Update to what we are currently running in production

This will also process wsgi log outputs from the projects
and grab the metadata (response code, time, request id).
You can use this to monitor the response time of the apis.

Change-Id: I0c061fa7c4ac07fe0789b51d17eee6e89ebe7f86
Kris Lindgren 3 years ago
parent
commit
a183507b44
1 changed files with 64 additions and 11 deletions
  1. 64
    11
      logstash/basic/logstash.conf

+ 64
- 11
logstash/basic/logstash.conf View File

@@ -159,18 +159,71 @@ filter {
159 159
       match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
160 160
       add_field => { "received_at" => "%{@timestamp}" }
161 161
     }
162
-
163
-  } else if "keystonefmt" in [tags] {
164
-    grok {
165
-      # Do multiline matching as the above mutliline filter may add newlines
166
-      # to the log messages.
167
-      # TODO move the LOGLEVELs into a proper grok pattern.
168
-      match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
169
-      add_field => { "received_at" => "%{@timestamp}" }
170
-    }
171 162
     if [module] == "iso8601.iso8601" {
172
-  #log message for each part of the date?  Really?
173
-  drop {}
163
+      drop {}
164
+    }
165
+
166
+    if "keystoneapi" in [tags] {
167
+       mutate {
168
+            gsub => ['logmessage',"\"",""]
169
+       }
170
+       grok {
171
+          match => { "logmessage" => "\[\-\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:method} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes} %{NUMBER:seconds}" }
172
+          add_field => ["api", "keystone"]
173
+          add_tag => ["apimetrics"]
174
+       }
175
+    } else if "novaapi" in [tags] {
176
+       if [module] == "nova.osapi_compute.wsgi.server" {
177
+         mutate {
178
+              gsub => ['logmessage',"\"",""]
179
+         }
180
+         grok {
181
+            match => { "logmessage" => "\[req\-%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant}\] %{NOTSPACE:requesterip} %{NOTSPACE:method} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes} time\: %{NUMBER:seconds}" }
182
+            add_field => ["api", "nova"]
183
+            add_tag => ["apimetrics"]
184
+         }
185
+       }
186
+    } else if "neutronapi" in [tags] {
187
+       if [module] == "neutron.wsgi" {
188
+         if "accepted" not in [logmessage] {
189
+           mutate {
190
+                gsub => ['logmessage',"\"",""]
191
+           }
192
+           grok {
193
+              match => { "logmessage" => "\[req\-%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant}\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:method} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes} %{NUMBER:seconds}" }
194
+              add_field => ["api", "neutron"]
195
+              add_tag => ["apimetrics"]
196
+           }
197
+         }
198
+       }
199
+    } else if "glanceapi" in [tags] {
200
+       if [module] == "glance.wsgi.server" {
201
+         mutate {
202
+              gsub => ['logmessage',"\"",""]
203
+         }
204
+         grok {
205
+            match => { "logmessage" => "\[%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:method} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes} %{NUMBER:seconds}" }
206
+            add_field => ["api", "glance"]
207
+            add_tag => ["apimetrics"]
208
+         }
209
+       }
210
+    } else if "novametaapi" in [tags] {
211
+       mutate {
212
+            gsub => ['logmessage',"\"",""]
213
+       }
214
+       if [module] == "nova.api.ec2" {
215
+         grok {
216
+            match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NUMBER:seconds}s %{NOTSPACE:requesterip} %{NOTSPACE:method} %{NOTSPACE:url_path} None\:None %{NUMBER:response} %{GREEDYDATA:user_agent}" }
217
+            add_field => ["api", "metadata-ec2"]
218
+            add_tag => ["apimetrics"]
219
+         }
220
+       } else if [module] == "nova.metadata.wsgi.server" {
221
+         grok {
222
+            match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NOTSPACE:requesterip} %{NOTSPACE:method} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes} time\: %{NUMBER:seconds}" }
223
+            add_field => ["api", "metadata"]
224
+            add_tag => ["apimetrics"]
225
+         }
226
+       }
174 227
     }
175 228
   } else if "libvirt" in [tags] {
176 229
     grok {

Loading…
Cancel
Save