Browse Source

Fluent-logging: Update fluentbit input definitions

This updates the fluent-logging chart to match the input configs
used in the upstream chart, primarily focused on moving from the
sqlite db file tracking mechanism to the in memory mechanism
instead. This also adds the appropriate fluentbit filter config
for throttling the number of records being forwarded to fluentd

Change-Id: I68ebf05b1b9fcf2a98b4717b74e3d13050446a09
tags/v1.0
Steve Wilkerson 2 months ago
parent
commit
84beb29ddd

+ 7
- 20
global/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml View File

@@ -150,9 +150,7 @@ data:
150 150
             Name: tail
151 151
             Tag: kernel
152 152
             Path: /var/log/kern.log
153
-            DB: /var/log/kern.db
154 153
             Mem_Buf_Limit: 5MB
155
-            DB.Sync: Normal
156 154
             Buffer_Chunk_Size: 1M
157 155
             Buffer_Max_Size: 1M
158 156
         - kubelet:
@@ -161,9 +159,7 @@ data:
161 159
             Tag: journal.*
162 160
             Path: ${JOURNAL_PATH}
163 161
             Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
164
-            DB: /var/log/kubelet.db
165 162
             Mem_Buf_Limit: 5MB
166
-            DB.Sync: Normal
167 163
             Buffer_Chunk_Size: 1M
168 164
             Buffer_Max_Size: 1M
169 165
         - docker_daemon:
@@ -172,9 +168,7 @@ data:
172 168
             Tag: journal.*
173 169
             Path: ${JOURNAL_PATH}
174 170
             Systemd_Filter: _SYSTEMD_UNIT=docker.service
175
-            DB: /var/log/docker.db
176 171
             Mem_Buf_Limit: 5MB
177
-            DB.Sync: Normal
178 172
             Buffer_Chunk_Size: 1M
179 173
             Buffer_Max_Size: 1M
180 174
         - libvirt:
@@ -182,9 +176,7 @@ data:
182 176
             Name: tail
183 177
             Tag: libvirt
184 178
             Path: /var/log/libvirt/libvirtd.log
185
-            DB: /var/log/libvirt.db
186 179
             Mem_Buf_Limit: 5MB
187
-            DB.Sync: Normal
188 180
             Buffer_Chunk_Size: 1M
189 181
             Buffer_Max_Size: 1M
190 182
         - qemu:
@@ -192,9 +184,7 @@ data:
192 184
             Name: tail
193 185
             Tag: qemu
194 186
             Path: /var/log/libvirt/qemu/*.log
195
-            DB: /var/log/qemu.db
196 187
             Mem_Buf_Limit: 5MB
197
-            DB.Sync: Normal
198 188
             Buffer_Chunk_Size: 1M
199 189
             Buffer_Max_Size: 1M
200 190
         - libvirt_record_modifier:
@@ -236,10 +226,8 @@ data:
236 226
             Name: tail
237 227
             Tag: ceph.cluster.*
238 228
             Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.log
239
-            DB: /var/log/ceph.db
240 229
             Parsers: syslog
241 230
             Mem_Buf_Limit: 5MB
242
-            DB.Sync: Normal
243 231
             Buffer_Chunk_Size: 1M
244 232
             Buffer_Max_Size: 1M
245 233
         - ceph_audit_logs:
@@ -247,10 +235,8 @@ data:
247 235
             Name: tail
248 236
             Tag: ceph.audit.*
249 237
             Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.audit.log
250
-            DB: /var/log/ceph.db
251 238
             Parsers: syslog
252 239
             Mem_Buf_Limit: 5MB
253
-            DB.Sync: Normal
254 240
             Buffer_Chunk_Size: 1M
255 241
             Buffer_Max_Size: 1M
256 242
         - ceph_mon_logs:
@@ -258,10 +244,8 @@ data:
258 244
             Name: tail
259 245
             Tag: ceph.mon.*
260 246
             Path: /var/log/ceph/airship-ucp-ceph-mon/ceph-mon**.log
261
-            DB: /var/log/ceph.db
262 247
             Parsers: syslog
263 248
             Mem_Buf_Limit: 5MB
264
-            DB.Sync: Normal
265 249
             Buffer_Chunk_Size: 1M
266 250
             Buffer_Max_Size: 1M
267 251
         - ceph_osd_logs:
@@ -269,10 +253,8 @@ data:
269 253
             Name: tail
270 254
             Tag: ceph.osd.*
271 255
             Path: /var/log/ceph/airship-ucp-ceph-osd/ceph-osd**.log
272
-            DB: /var/log/ceph.db
273 256
             Parsers: syslog
274 257
             Mem_Buf_Limit: 5MB
275
-            DB.Sync: Normal
276 258
             Buffer_Chunk_Size: 1M
277 259
             Buffer_Max_Size: 1M
278 260
         - containers_tail:
@@ -281,8 +263,6 @@ data:
281 263
             Tag: kube.*
282 264
             Path: /var/log/containers/*.log
283 265
             Parser: docker
284
-            DB: /var/log/flb_kube.db
285
-            DB.Sync: Normal
286 266
             Buffer_Chunk_Size: 1M
287 267
             Buffer_Max_Size: 1M
288 268
             Mem_Buf_Limit: 5MB
@@ -295,6 +275,13 @@ data:
295 275
             Name: kubernetes
296 276
             Match: kube.*
297 277
             Merge_JSON_Log: On
278
+        - throttle_filter:
279
+            header: filter
280
+            Name: throttle
281
+            Match: "**"
282
+            Rate: 1000
283
+            Window: 300
284
+            Interval: 1s
298 285
         - fluentd_output:
299 286
             header: output
300 287
             Name: forward

Loading…
Cancel
Save