Browse Source

Basic logstash and kibana dashboard configs

May not [probably won't] be applicable in all environments.
Encourage others to submit PRs against this to make it a more
generic config that should work for most setups.
Mike Dorman 4 years ago
parent
commit
173fda225d
2 changed files with 657 additions and 0 deletions
  1. 452
    0
      logstash/kibana.json
  2. 205
    0
      logstash/logstash.conf

+ 452
- 0
logstash/kibana.json View File

@@ -0,0 +1,452 @@
1
+{
2
+  "title": "Openstack Logs",
3
+  "services": {
4
+    "query": {
5
+      "list": {
6
+        "0": {
7
+          "id": 0,
8
+          "color": "#7EB26D",
9
+          "alias": "INFO",
10
+          "pin": false,
11
+          "type": "lucene",
12
+          "enable": true,
13
+          "query": "loglevel:INFO"
14
+        },
15
+        "1": {
16
+          "id": 1,
17
+          "color": "#EAB839",
18
+          "alias": "TRACE",
19
+          "pin": false,
20
+          "type": "lucene",
21
+          "enable": true,
22
+          "query": "loglevel:TRACE"
23
+        },
24
+        "2": {
25
+          "id": 2,
26
+          "color": "#6ED0E0",
27
+          "alias": "DEBUG",
28
+          "pin": false,
29
+          "type": "lucene",
30
+          "enable": true,
31
+          "query": "loglevel:DEBUG"
32
+        },
33
+        "3": {
34
+          "id": 3,
35
+          "color": "#5195CE",
36
+          "alias": "AUDIT",
37
+          "pin": false,
38
+          "type": "lucene",
39
+          "enable": true,
40
+          "query": "loglevel:AUDIT"
41
+        },
42
+        "4": {
43
+          "id": 4,
44
+          "color": "#C15C17",
45
+          "alias": "WARNING",
46
+          "pin": false,
47
+          "type": "lucene",
48
+          "enable": true,
49
+          "query": "loglevel:WARNING"
50
+        },
51
+        "9": {
52
+          "id": 9,
53
+          "type": "lucene",
54
+          "query": "loglevel:ERROR",
55
+          "alias": "ERROR",
56
+          "color": "#BF1B00",
57
+          "pin": false,
58
+          "enable": true
59
+        }
60
+      },
61
+      "ids": [
62
+        0,
63
+        1,
64
+        2,
65
+        3,
66
+        4,
67
+        9
68
+      ]
69
+    },
70
+    "filter": {
71
+      "list": {
72
+        "0": {
73
+          "type": "time",
74
+          "field": "@timestamp",
75
+          "from": "now-1h",
76
+          "to": "now",
77
+          "mandate": "must",
78
+          "active": true,
79
+          "alias": "",
80
+          "id": 0
81
+        }
82
+      },
83
+      "ids": [
84
+        0
85
+      ]
86
+    }
87
+  },
88
+  "rows": [
89
+    {
90
+      "title": "Histogram",
91
+      "height": "150px",
92
+      "editable": true,
93
+      "collapse": false,
94
+      "collapsable": true,
95
+      "panels": [
96
+        {
97
+          "span": 12,
98
+          "editable": true,
99
+          "type": "histogram",
100
+          "loadingEditor": false,
101
+          "mode": "count",
102
+          "time_field": "@timestamp",
103
+          "value_field": null,
104
+          "x-axis": true,
105
+          "y-axis": true,
106
+          "scale": 1,
107
+          "y_format": "none",
108
+          "grid": {
109
+            "max": null,
110
+            "min": 0
111
+          },
112
+          "queries": {
113
+            "mode": "all",
114
+            "ids": [
115
+              0,
116
+              1,
117
+              2,
118
+              3,
119
+              4,
120
+              9
121
+            ]
122
+          },
123
+          "annotate": {
124
+            "enable": false,
125
+            "query": "*",
126
+            "size": 20,
127
+            "field": "_type",
128
+            "sort": [
129
+              "_score",
130
+              "desc"
131
+            ]
132
+          },
133
+          "auto_int": true,
134
+          "resolution": 100,
135
+          "interval": "30s",
136
+          "intervals": [
137
+            "auto",
138
+            "1s",
139
+            "1m",
140
+            "5m",
141
+            "10m",
142
+            "30m",
143
+            "1h",
144
+            "3h",
145
+            "12h",
146
+            "1d",
147
+            "1w",
148
+            "1y"
149
+          ],
150
+          "lines": false,
151
+          "fill": 0,
152
+          "linewidth": 3,
153
+          "points": false,
154
+          "pointradius": 5,
155
+          "bars": true,
156
+          "stack": true,
157
+          "spyable": true,
158
+          "zoomlinks": true,
159
+          "options": true,
160
+          "legend": true,
161
+          "show_query": true,
162
+          "interactive": true,
163
+          "legend_counts": true,
164
+          "timezone": "browser",
165
+          "percentage": false,
166
+          "zerofill": true,
167
+          "derivative": false,
168
+          "tooltip": {
169
+            "value_type": "cumulative",
170
+            "query_as_alias": true
171
+          },
172
+          "title": "Events by Time",
173
+          "scaleSeconds": false
174
+        }
175
+      ],
176
+      "notice": false
177
+    },
178
+    {
179
+      "title": "Graph",
180
+      "height": "350px",
181
+      "editable": true,
182
+      "collapse": false,
183
+      "collapsable": true,
184
+      "panels": [
185
+        {
186
+          "error": false,
187
+          "span": 4,
188
+          "editable": true,
189
+          "type": "terms",
190
+          "loadingEditor": false,
191
+          "field": "loglevel",
192
+          "exclude": [
193
+            ""
194
+          ],
195
+          "missing": false,
196
+          "other": false,
197
+          "size": 18,
198
+          "order": "count",
199
+          "style": {
200
+            "font-size": "10pt"
201
+          },
202
+          "donut": false,
203
+          "tilt": false,
204
+          "labels": true,
205
+          "arrangement": "horizontal",
206
+          "chart": "table",
207
+          "counter_pos": "above",
208
+          "spyable": true,
209
+          "queries": {
210
+            "mode": "all",
211
+            "ids": [
212
+              0,
213
+              1,
214
+              2,
215
+              3,
216
+              4,
217
+              9
218
+            ]
219
+          },
220
+          "title": "Events",
221
+          "tmode": "terms",
222
+          "tstat": "total",
223
+          "valuefield": ""
224
+        },
225
+        {
226
+          "error": false,
227
+          "span": 4,
228
+          "editable": true,
229
+          "type": "terms",
230
+          "loadingEditor": false,
231
+          "field": "host",
232
+          "exclude": [],
233
+          "missing": false,
234
+          "other": false,
235
+          "size": 10,
236
+          "order": "count",
237
+          "style": {
238
+            "font-size": "10pt"
239
+          },
240
+          "donut": false,
241
+          "tilt": false,
242
+          "labels": true,
243
+          "arrangement": "horizontal",
244
+          "chart": "table",
245
+          "counter_pos": "above",
246
+          "spyable": true,
247
+          "queries": {
248
+            "mode": "all",
249
+            "ids": [
250
+              0,
251
+              1,
252
+              2,
253
+              3,
254
+              4,
255
+              9
256
+            ]
257
+          },
258
+          "title": "Events by Host",
259
+          "tmode": "terms",
260
+          "tstat": "total",
261
+          "valuefield": ""
262
+        },
263
+        {
264
+          "error": false,
265
+          "span": 4,
266
+          "editable": true,
267
+          "type": "terms",
268
+          "loadingEditor": false,
269
+          "field": "_type",
270
+          "exclude": [],
271
+          "missing": false,
272
+          "other": false,
273
+          "size": 10,
274
+          "order": "count",
275
+          "style": {
276
+            "font-size": "10pt"
277
+          },
278
+          "donut": false,
279
+          "tilt": false,
280
+          "labels": true,
281
+          "arrangement": "horizontal",
282
+          "chart": "pie",
283
+          "counter_pos": "above",
284
+          "spyable": true,
285
+          "queries": {
286
+            "mode": "all",
287
+            "ids": [
288
+              0,
289
+              1,
290
+              2,
291
+              3,
292
+              4,
293
+              9
294
+            ]
295
+          },
296
+          "title": "Events Type",
297
+          "tmode": "terms",
298
+          "tstat": "total",
299
+          "valuefield": ""
300
+        }
301
+      ],
302
+      "notice": false
303
+    },
304
+    {
305
+      "title": "Events",
306
+      "height": "350px",
307
+      "editable": true,
308
+      "collapse": false,
309
+      "collapsable": true,
310
+      "panels": [
311
+        {
312
+          "title": "All events",
313
+          "error": false,
314
+          "span": 12,
315
+          "editable": true,
316
+          "group": [
317
+            "default"
318
+          ],
319
+          "type": "table",
320
+          "size": 100,
321
+          "pages": 5,
322
+          "offset": 0,
323
+          "sort": [
324
+            "@timestamp",
325
+            "desc"
326
+          ],
327
+          "style": {
328
+            "font-size": "8pt"
329
+          },
330
+          "overflow": "min-height",
331
+          "fields": [
332
+            "@timestamp",
333
+            "host",
334
+            "loglevel",
335
+            "_type",
336
+            "module",
337
+            "logmessage"
338
+          ],
339
+          "localTime": true,
340
+          "timeField": "@timestamp",
341
+          "highlight": [],
342
+          "sortable": true,
343
+          "header": true,
344
+          "paging": true,
345
+          "spyable": true,
346
+          "queries": {
347
+            "mode": "all",
348
+            "ids": [
349
+              0,
350
+              1,
351
+              2,
352
+              3,
353
+              4,
354
+              9
355
+            ]
356
+          },
357
+          "field_list": false,
358
+          "status": "Stable",
359
+          "trimFactor": 700,
360
+          "normTimes": true,
361
+          "all_fields": false
362
+        }
363
+      ],
364
+      "notice": false
365
+    }
366
+  ],
367
+  "editable": true,
368
+  "failover": false,
369
+  "index": {
370
+    "interval": "day",
371
+    "pattern": "[logstash-]YYYY.MM.DD",
372
+    "default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED",
373
+    "warm_fields": true
374
+  },
375
+  "style": "dark",
376
+  "panel_hints": true,
377
+  "pulldowns": [
378
+    {
379
+      "type": "query",
380
+      "collapse": true,
381
+      "notice": false,
382
+      "query": "*",
383
+      "pinned": true,
384
+      "history": [
385
+        "loglevel:ERROR",
386
+        "loglevel:WARNING",
387
+        "loglevel:AUDIT",
388
+        "loglevel:DEBUG",
389
+        "loglevel:TRACE",
390
+        "loglevel:INFO",
391
+      ],
392
+      "remember": 10,
393
+      "enable": true
394
+    },
395
+    {
396
+      "type": "filtering",
397
+      "collapse": false,
398
+      "notice": true,
399
+      "enable": true
400
+    }
401
+  ],
402
+  "nav": [
403
+    {
404
+      "type": "timepicker",
405
+      "collapse": false,
406
+      "notice": false,
407
+      "status": "Stable",
408
+      "time_options": [
409
+        "5m",
410
+        "15m",
411
+        "1h",
412
+        "6h",
413
+        "12h",
414
+        "24h",
415
+        "2d",
416
+        "7d",
417
+        "30d"
418
+      ],
419
+      "refresh_intervals": [
420
+        "5s",
421
+        "10s",
422
+        "30s",
423
+        "1m",
424
+        "5m",
425
+        "15m",
426
+        "30m",
427
+        "1h",
428
+        "2h",
429
+        "1d"
430
+      ],
431
+      "timefield": "@timestamp",
432
+      "now": true,
433
+      "filter_id": 0,
434
+      "enable": true
435
+    }
436
+  ],
437
+  "loader": {
438
+    "save_gist": true,
439
+    "save_elasticsearch": true,
440
+    "save_local": true,
441
+    "save_default": true,
442
+    "save_temp": true,
443
+    "save_temp_ttl_enable": true,
444
+    "save_temp_ttl": "30d",
445
+    "load_gist": true,
446
+    "load_elasticsearch": true,
447
+    "load_elasticsearch_size": 20,
448
+    "load_local": true,
449
+    "hide": false
450
+  },
451
+  "refresh": "1m"
452
+}

+ 205
- 0
logstash/logstash.conf View File

@@ -0,0 +1,205 @@
1
+# logstash.conf
2
+#
3
+# Basic logstash config and filters for injesting most logs from Nova,
4
+# Keystone, Glance, Ceilometer, Heat, and Neutron services, as well as
5
+# Apache (Horizon) and syslog.
6
+#
7
+# Author: Kris Lindgren <klindgren@godaddy.com>
8
+#
9
+# Copyright (c) 2014 Go Daddy Operating Company, LLC
10
+#
11
+# Permission is hereby granted, free of charge, to any person obtaining a 
12
+# copy of this software and associated documentation files (the "Software"), 
13
+# to deal in the Software without restriction, including without limitation 
14
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, 
15
+# and/or sell copies of the Software, and to permit persons to whom the 
16
+# Software is furnished to do so, subject to the following conditions:
17
+#
18
+# The above copyright notice and this permission notice shall be included in 
19
+# all copies or substantial portions of the Software.
20
+#
21
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
22
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
23
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
24
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
25
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
26
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
27
+# DEALINGS IN THE SOFTWARE.
28
+#
29
+
30
+input {
31
+  file {
32
+    path => ['/var/log/nova/nova-api.log']
33
+    tags => ['nova', 'oslofmt']
34
+    type => "nova"
35
+  }
36
+  file {
37
+    path => ['/var/log/nova/nova-conductor.log']
38
+    tags => ['nova', 'oslofmt']
39
+    type => "nova"
40
+  }
41
+  file {
42
+    path => ['/var/log/nova/nova-manage.log']
43
+    tags => ['nova', 'oslofmt']
44
+    type => "nova"
45
+  }
46
+  file {
47
+    path => ['/var/log/nova/nova-scheduler.log']
48
+    tags => ['nova', 'oslofmt']
49
+    type => "nova"
50
+  }
51
+  file {
52
+    path => ['/var/log/nova/nova-spicehtml5proxy.log']
53
+    tags => ['nova', 'oslofmt']
54
+    type => "nova"
55
+  }
56
+  file {
57
+    path => ['/var/log/keystone/keystone-all.log']
58
+    tags => ['keystone', 'keystonefmt']
59
+    type => "keystone"
60
+  }
61
+  file {
62
+    path => ['/var/log/keystone/keystone-manage.log']
63
+    tags => ['keystone', 'keystonefmt']
64
+    type => "keystone"
65
+  }
66
+  file {
67
+    path => ['/var/log/glance/api.log']
68
+    tags => ['glance', 'oslofmt']
69
+    type => "glance"
70
+  }
71
+  file {
72
+    path => ['/var/log/glance/registry.log']
73
+    tags => ['glance', 'oslofmt']
74
+    type => "glance"
75
+  }
76
+  file {
77
+    path => ['/var/log/glance/scrubber.log']
78
+    tags => ['glance', 'oslofmt']
79
+    type => "glance"
80
+  }
81
+  file {
82
+    path => ['/var/log/ceilometer/ceilometer-agent-central.log']
83
+    tags => ['ceilometer', 'oslofmt']
84
+    type => "ceilometer"
85
+  }
86
+  file {
87
+    path => ['/var/log/ceilometer/ceilometer-alarm-notifier.log']
88
+    tags => ['ceilometer', 'oslofmt']
89
+    type => "ceilometer"
90
+  }
91
+  file {
92
+    path => ['/var/log/ceilometer/ceilometer-api.log']
93
+    tags => ['ceilometer', 'oslofmt']
94
+    type => "ceilometer"
95
+  }
96
+  file {
97
+    path => ['/var/log/ceilometer/ceilometer-alarm-evaluator.log']
98
+    tags => ['ceilometer', 'oslofmt']
99
+    type => "ceilometer"
100
+  }
101
+  file {
102
+    path => ['/var/log/ceilometer/ceilometer-collector.log']
103
+    tags => ['ceilometer', 'oslofmt']
104
+    type => "ceilometer"
105
+  }
106
+  file {
107
+    path => ['/var/log/heat/heat.log']
108
+    tags => ['heat', 'oslofmt']
109
+    type => "heat"
110
+  }
111
+  file {
112
+    path => ['/var/log/neutron/neutron-server.log']
113
+    tags => ['neutron', 'oslofmt']
114
+    type => "neutron"
115
+  }
116
+# Not collecting RabbitMQ logs for the moment
117
+#  file {
118
+#    path => ['/var/log/rabbitmq/rabbit@<%= @hostname %>.log']
119
+#    tags => ['rabbitmq', 'oslofmt']
120
+#    type => "rabbitmq"
121
+#  }
122
+  file {
123
+    path => ['/var/log/httpd/access_log']
124
+    tags => ['horizon']
125
+    type => "horizon"
126
+  }
127
+  file {
128
+    path => ['/var/log/httpd/error_log']
129
+    tags => ['horizon']
130
+    type => "horizon"
131
+  }
132
+  file {
133
+    path => ['/var/log/httpd/horizon_access_log']
134
+    tags => ['horizon']
135
+    type => "horizon"
136
+  }
137
+  file {
138
+    path => ['/var/log/httpd/horizon_error_log']
139
+    tags => ['horizon']
140
+    type => "horizon"
141
+  }
142
+}
143
+filter {
144
+  if "oslofmt" in [tags] {
145
+    multiline {
146
+      negate => true
147
+      pattern => "^%{TIMESTAMP_ISO8601} "
148
+      what => "previous"
149
+    }
150
+    multiline {
151
+      negate => false
152
+      pattern => "^%{TIMESTAMP_ISO8601}%{SPACE}%{NUMBER}?%{SPACE}?TRACE"
153
+      what => "previous"
154
+    }
155
+    grok {
156
+      # Do multiline matching as the above mutliline filter may add newlines
157
+      # to the log messages.
158
+      # TODO move the LOGLEVELs into a proper grok pattern.
159
+      match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
160
+      add_field => { "received_at" => "%{@timestamp}" }
161
+    }
162
+
163
+  } else if "keystonefmt" in [tags] {
164
+    grok {
165
+      # Do multiline matching as the above mutliline filter may add newlines
166
+      # to the log messages.
167
+      # TODO move the LOGLEVELs into a proper grok pattern.
168
+      match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
169
+      add_field => { "received_at" => "%{@timestamp}" }
170
+    }
171
+    if [module] == "iso8601.iso8601" {
172
+  #log message for each part of the date?  Really?
173
+  drop {}
174
+    }
175
+  } else if "libvirt" in [tags] {
176
+    grok {
177
+       match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}:%{SPACE}%{NUMBER:code}:?%{SPACE}\[?\b%{NOTSPACE:loglevel}\b\]?%{SPACE}?:?%{SPACE}\[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
178
+       add_field => { "received_at" => "%{@timestamp}"}
179
+    }
180
+    mutate {
181
+       uppercase => [ "loglevel" ]
182
+    }
183
+  } else if [type] == "syslog" {
184
+     grok {
185
+        match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:logmessage}" }
186
+        add_field => [ "received_at", "%{@timestamp}" ]
187
+     }
188
+     syslog_pri {
189
+        severity_labels => ["ERROR", "ERROR", "ERROR", "ERROR", "WARNING", "INFO", "INFO", "DEBUG" ]
190
+     }
191
+     date {
192
+        match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
193
+     }
194
+     if !("_grokparsefailure" in [tags]) {
195
+        mutate {
196
+           replace => [ "@source_host", "%{syslog_hostname}" ]
197
+        }
198
+     }
199
+     mutate {
200
+        remove_field => [ "syslog_hostname", "syslog_timestamp" ]
201
+        add_field => [ "loglevel", "%{syslog_severity}" ]
202
+        add_field => [ "module", "%{syslog_program}" ]
203
+     }
204
+  }
205
+}

Loading…
Cancel
Save