Browse Source

Uplift and Separate fluentbit and fluentd charts

fluent-logging chart was splitter into fluentbit and fluentd charts
in the following PS: https://review.opendev.org/642146

Change-Id: Ic72f9fb6293b99395a0acf48064b9c04e312d365
tags/v1.2
Alexander Noskov 3 months ago
parent
commit
87b26958a4

+ 3
- 1
doc/source/airsloop.rst View File

@@ -415,7 +415,9 @@ Under site/airsloop/software/config/ create a versions.yaml file in the followin
415 415
         osh_infra:
416 416
           elasticsearch:
417 417
             proxy_server: proxy.example.com:8080
418
-          fluent_logging:
418
+          fluentbit:
419
+            proxy_server: proxy.example.com:8080
420
+          fluentd:
419 421
             proxy_server: proxy.example.com:8080
420 422
           grafana:
421 423
             proxy_server: proxy.example.com:8080

+ 12
- 1
global/schemas/pegleg/SoftwareVersions/v1.yaml View File

@@ -235,7 +235,18 @@ data:
235 235
                   type: string
236 236
                 reference:
237 237
                   type: string
238
-            fluent_logging:
238
+            fluentbit:
239
+              type: object
240
+              properties:
241
+                type:
242
+                  type: string
243
+                location:
244
+                  type: string
245
+                subpath:
246
+                  type: string
247
+                reference:
248
+                  type: string
249
+            fluentd:
239 250
               type: object
240 251
               properties:
241 252
                 type:

+ 2
- 1
global/software/charts/osh-infra/osh-infra-logging/chart-group.yaml View File

@@ -12,4 +12,5 @@ data:
12 12
   sequenced: True
13 13
   chart_group:
14 14
     - elasticsearch
15
-    - fluent-logging
15
+    - fluentbit
16
+    - fluentd

+ 0
- 419
global/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml View File

@@ -1,419 +0,0 @@
1
----
2
-schema: armada/Chart/v1
3
-metadata:
4
-  schema: metadata/Document/v1
5
-  name: fluent-logging-global
6
-  layeringDefinition:
7
-    abstract: true
8
-    layer: global
9
-  labels:
10
-    hosttype: fluent-logging-global
11
-  storagePolicy: cleartext
12
-  substitutions:
13
-    # Chart source
14
-    - src:
15
-        schema: pegleg/SoftwareVersions/v1
16
-        name: software-versions
17
-        path: .charts.osh_infra.fluent_logging
18
-      dest:
19
-        path: .source
20
-
21
-    # Images
22
-    - src:
23
-        schema: pegleg/SoftwareVersions/v1
24
-        name: software-versions
25
-        path: .images.osh_infra.fluent_logging
26
-      dest:
27
-        path: .values.images.tags
28
-
29
-    # Endpoints
30
-    - src:
31
-        schema: pegleg/EndpointCatalogue/v1
32
-        name: osh_infra_endpoints
33
-        path: .osh_infra.elasticsearch
34
-      dest:
35
-        path: .values.endpoints.elasticsearch
36
-    - src:
37
-        schema: pegleg/EndpointCatalogue/v1
38
-        name: osh_infra_endpoints
39
-        path: .osh_infra.fluentd
40
-      dest:
41
-        path: .values.endpoints.fluentd
42
-    - src:
43
-        schema: pegleg/EndpointCatalogue/v1
44
-        name: osh_infra_endpoints
45
-        path: .osh_infra.prometheus_fluentd_exporter
46
-      dest:
47
-        path: .values.endpoints.prometheus_fluentd_exporter
48
-    # Accounts
49
-    - src:
50
-        schema: pegleg/AccountCatalogue/v1
51
-        name: osh_infra_service_accounts
52
-        path: .osh_infra.elasticsearch.admin
53
-      dest:
54
-        path: .values.endpoints.elasticsearch.auth.admin
55
-
56
-    # Secrets
57
-    - dest:
58
-        path: .values.endpoints.elasticsearch.auth.admin.password
59
-      src:
60
-        schema: deckhand/Passphrase/v1
61
-        name: osh_infra_elasticsearch_admin_password
62
-        path: .
63
-
64
-data:
65
-  chart_name: fluent-logging
66
-  release: fluent-logging
67
-  namespace: osh-infra
68
-  wait:
69
-    timeout: 900
70
-    labels:
71
-      release_group: airship-fluent-logging
72
-  install:
73
-    no_hooks: false
74
-  upgrade:
75
-    no_hooks: false
76
-    pre:
77
-      delete:
78
-        - type: job
79
-          labels:
80
-            release_group: airship-fluent-logging
81
-      create: []
82
-    post:
83
-      create: []
84
-  values:
85
-    monitoring:
86
-      prometheus:
87
-        enabled: true
88
-    pod:
89
-      resources:
90
-        enabled: true
91
-        fluentbit:
92
-          limits:
93
-            memory: '4Gi'
94
-            cpu: '2000m'
95
-          requests:
96
-            memory: '2Gi'
97
-            cpu: '1000m'
98
-        fluentd:
99
-          limits:
100
-            memory: '4Gi'
101
-            cpu: '2000m'
102
-          requests:
103
-            memory: '2Gi'
104
-            cpu: '1000m'
105
-        prometheus_fluentd_exporter:
106
-          limits:
107
-            memory: '1024Mi'
108
-            cpu: '2000m'
109
-          requests:
110
-            memory: '0'
111
-            cpu: '0'
112
-        jobs:
113
-          image_repo_sync:
114
-            requests:
115
-              memory: '0'
116
-              cpu: '0'
117
-            limits:
118
-              memory: '1024Mi'
119
-              cpu: '2000m'
120
-          tests:
121
-            requests:
122
-              memory: '0'
123
-              cpu: '0'
124
-            limits:
125
-              memory: '1024Mi'
126
-              cpu: '2000m'
127
-    labels:
128
-      fluentd:
129
-        node_selector_key: fluentd
130
-        node_selector_value: enabled
131
-      fluentbit:
132
-        node_selector_key: fluentbit
133
-        node_selector_value: enabled
134
-      prometheus_fluentd_exporter:
135
-        node_selector_key: openstack-control-plane
136
-        node_selector_value: enabled
137
-      job:
138
-        node_selector_key: openstack-control-plane
139
-        node_selector_value: enabled
140
-    conf:
141
-      fluentbit:
142
-        - service:
143
-            header: service
144
-            Flush: 5
145
-            Daemon: Off
146
-            Log_Level: info
147
-            Parsers_File: parsers.conf
148
-        - kernel_messages:
149
-            header: input
150
-            Name: tail
151
-            Tag: kernel
152
-            Path: /var/log/kern.log
153
-            Mem_Buf_Limit: 5MB
154
-            Buffer_Chunk_Size: 1M
155
-            Buffer_Max_Size: 1M
156
-        - kubelet:
157
-            header: input
158
-            Name: systemd
159
-            Tag: journal.*
160
-            Path: ${JOURNAL_PATH}
161
-            Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
162
-            Mem_Buf_Limit: 5MB
163
-            Buffer_Chunk_Size: 1M
164
-            Buffer_Max_Size: 1M
165
-        - docker_daemon:
166
-            header: input
167
-            Name: systemd
168
-            Tag: journal.*
169
-            Path: ${JOURNAL_PATH}
170
-            Systemd_Filter: _SYSTEMD_UNIT=docker.service
171
-            Mem_Buf_Limit: 5MB
172
-            Buffer_Chunk_Size: 1M
173
-            Buffer_Max_Size: 1M
174
-        - libvirt:
175
-            header: input
176
-            Name: tail
177
-            Tag: libvirt
178
-            Path: /var/log/libvirt/libvirtd.log
179
-            Mem_Buf_Limit: 5MB
180
-            Buffer_Chunk_Size: 1M
181
-            Buffer_Max_Size: 1M
182
-        - qemu:
183
-            header: input
184
-            Name: tail
185
-            Tag: qemu
186
-            Path: /var/log/libvirt/qemu/*.log
187
-            Mem_Buf_Limit: 5MB
188
-            Buffer_Chunk_Size: 1M
189
-            Buffer_Max_Size: 1M
190
-        - libvirt_record_modifier:
191
-            header: filter
192
-            Name: record_modifier
193
-            Match: libvirt
194
-            Record: hostname ${HOSTNAME}
195
-        - qemu_record_modifier:
196
-            header: filter
197
-            Name: record_modifier
198
-            Match: qemu
199
-            Record: hostname ${HOSTNAME}
200
-        - kernel_record_modifier:
201
-            header: filter
202
-            Name: record_modifier
203
-            Match: kernel
204
-            Record: hostname ${HOSTNAME}
205
-        - systemd_modify_fields:
206
-            header: filter
207
-            Name: modify
208
-            Match: journal.**
209
-            Rename:
210
-              _BOOT_ID: BOOT_ID
211
-              _CAP_EFFECTIVE: CAP_EFFECTIVE
212
-              _CMDLINE: CMDLINE
213
-              _COMM: COMM
214
-              _EXE: EXE
215
-              _GID: GID
216
-              _HOSTNAME: HOSTNAME
217
-              _MACHINE_ID: MACHINE_ID
218
-              _PID: PID
219
-              _SYSTEMD_CGROUP: SYSTEMD_CGROUP
220
-              _SYSTEMD_SLICE: SYSTEMD_SLICE
221
-              _SYSTEMD_UNIT: SYSTEMD_UNIT
222
-              _UID: UID
223
-              _TRANSPORT: TRANSPORT
224
-        - ceph_cluster_logs:
225
-            header: input
226
-            Name: tail
227
-            Tag: ceph.cluster.*
228
-            Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.log
229
-            Parsers: syslog
230
-            Mem_Buf_Limit: 5MB
231
-            Buffer_Chunk_Size: 1M
232
-            Buffer_Max_Size: 1M
233
-        - ceph_audit_logs:
234
-            header: input
235
-            Name: tail
236
-            Tag: ceph.audit.*
237
-            Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.audit.log
238
-            Parsers: syslog
239
-            Mem_Buf_Limit: 5MB
240
-            Buffer_Chunk_Size: 1M
241
-            Buffer_Max_Size: 1M
242
-        - ceph_mon_logs:
243
-            header: input
244
-            Name: tail
245
-            Tag: ceph.mon.*
246
-            Path: /var/log/ceph/airship-ucp-ceph-mon/ceph-mon**.log
247
-            Parsers: syslog
248
-            Mem_Buf_Limit: 5MB
249
-            Buffer_Chunk_Size: 1M
250
-            Buffer_Max_Size: 1M
251
-        - ceph_osd_logs:
252
-            header: input
253
-            Name: tail
254
-            Tag: ceph.osd.*
255
-            Path: /var/log/ceph/airship-ucp-ceph-osd/ceph-osd**.log
256
-            Parsers: syslog
257
-            Mem_Buf_Limit: 5MB
258
-            Buffer_Chunk_Size: 1M
259
-            Buffer_Max_Size: 1M
260
-        - containers_tail:
261
-            header: input
262
-            Name: tail
263
-            Tag: kube.*
264
-            Path: /var/log/containers/*.log
265
-            Parser: docker
266
-            Buffer_Chunk_Size: 1M
267
-            Buffer_Max_Size: 1M
268
-            Mem_Buf_Limit: 5MB
269
-        - drop_fluentd_logs:
270
-            header: output
271
-            Name: "null"
272
-            Match: "**.fluentd**"
273
-        - kube_filter:
274
-            header: filter
275
-            Name: kubernetes
276
-            Match: kube.*
277
-            Merge_JSON_Log: On
278
-        - throttle_filter:
279
-            header: filter
280
-            Name: throttle
281
-            Match: "**"
282
-            Rate: 1000
283
-            Window: 300
284
-            Interval: 1s
285
-        - fluentd_output:
286
-            header: output
287
-            Name: forward
288
-            Match: "*"
289
-            Host: ${FLUENTD_HOST}
290
-            Port: ${FLUENTD_PORT}
291
-      fluentd:
292
-        - metrics_agent:
293
-            header: source
294
-            type: monitor_agent
295
-            bind: 0.0.0.0
296
-            port: 24220
297
-        - fluentbit_forward:
298
-            header: source
299
-            type: forward
300
-            port: "#{ENV['FLUENTD_PORT']}"
301
-            bind: 0.0.0.0
302
-        - filter_fluentd_logs:
303
-            header: match
304
-            expression: "fluent.**"
305
-            type: "null"
306
-        - journal_elasticsearch:
307
-            header: match
308
-            type: elasticsearch
309
-            user: "#{ENV['ELASTICSEARCH_USERNAME']}"
310
-            password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
311
-            expression: "journal.**"
312
-            include_tag_key: true
313
-            host: "#{ENV['ELASTICSEARCH_HOST']}"
314
-            port: "#{ENV['ELASTICSEARCH_PORT']}"
315
-            logstash_format: true
316
-            logstash_prefix: journal
317
-            buffer:
318
-              -
319
-                - header: buffer
320
-                  flush_thread_count: 8
321
-                  flush_interval: "15"
322
-                  chunk_limit_size: 8MB
323
-                  queue_limit_length: 256
324
-                  retry_max_interval: 30
325
-                  retry_forever: false
326
-        - kernel_elasticsearch:
327
-            header: match
328
-            type: elasticsearch
329
-            user: "#{ENV['ELASTICSEARCH_USERNAME']}"
330
-            password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
331
-            expression: "kernel"
332
-            include_tag_key: true
333
-            host: "#{ENV['ELASTICSEARCH_HOST']}"
334
-            port: "#{ENV['ELASTICSEARCH_PORT']}"
335
-            logstash_format: true
336
-            logstash_prefix: kernel
337
-            buffer:
338
-              -
339
-                - header: buffer
340
-                  flush_thread_count: 8
341
-                  flush_interval: "15"
342
-                  chunk_limit_size: 8MB
343
-                  queue_limit_length: 256
344
-                  retry_max_interval: 30
345
-                  retry_forever: false
346
-        - ceph_elasticsearch:
347
-            header: match
348
-            type: elasticsearch
349
-            user: "#{ENV['ELASTICSEARCH_USERNAME']}"
350
-            password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
351
-            expression: "**ceph-**.log"
352
-            include_tag_key: true
353
-            host: "#{ENV['ELASTICSEARCH_HOST']}"
354
-            port: "#{ENV['ELASTICSEARCH_PORT']}"
355
-            logstash_format: true
356
-            logstash_prefix: ceph
357
-            buffer:
358
-              -
359
-                - header: buffer
360
-                  flush_thread_count: 8
361
-                  flush_interval: "15"
362
-                  chunk_limit_size: 8MB
363
-                  queue_limit_length: 256
364
-                  retry_max_interval: 30
365
-                  retry_forever: false
366
-        - oslo_fluentd_elasticsearch:
367
-            header: match
368
-            type: elasticsearch
369
-            user: "#{ENV['ELASTICSEARCH_USERNAME']}"
370
-            password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
371
-            expression: "**.openstack.*"
372
-            include_tag_key: true
373
-            host: "#{ENV['ELASTICSEARCH_HOST']}"
374
-            port: "#{ENV['ELASTICSEARCH_PORT']}"
375
-            logstash_format: true
376
-            logstash_prefix: openstack
377
-            buffer:
378
-              -
379
-                - header: buffer
380
-                  flush_thread_count: 8
381
-                  flush_interval: "15"
382
-                  chunk_limit_size: 8MB
383
-                  queue_limit_length: 256
384
-                  retry_max_interval: 30
385
-                  retry_forever: false
386
-            type_name: oslo_openstack_fluentd
387
-        - elasticsearch:
388
-            header: match
389
-            type: elasticsearch
390
-            user: "#{ENV['ELASTICSEARCH_USERNAME']}"
391
-            password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
392
-            expression: "**"
393
-            include_tag_key: true
394
-            host: "#{ENV['ELASTICSEARCH_HOST']}"
395
-            port: "#{ENV['ELASTICSEARCH_PORT']}"
396
-            logstash_format: true
397
-            buffer:
398
-              -
399
-                - header: buffer
400
-                  flush_thread_count: 8
401
-                  flush_interval: "15"
402
-                  chunk_limit_size: 8MB
403
-                  queue_limit_length: 256
404
-                  retry_max_interval: 30
405
-                  retry_forever: false
406
-            flush_interval: "10"
407
-      parsers:
408
-        - syslog:
409
-            header: parser
410
-            Name: syslog
411
-            Format: regex
412
-            Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
413
-            Time_Key: time
414
-            Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
415
-            Time_Keep: On
416
-            Types: "pid:integer"
417
-  dependencies:
418
-    - osh-infra-helm-toolkit
419
-...

+ 255
- 0
global/software/charts/osh-infra/osh-infra-logging/fluentbit.yaml View File

@@ -0,0 +1,255 @@
1
+---
2
+schema: armada/Chart/v1
3
+metadata:
4
+  schema: metadata/Document/v1
5
+  name: fluentbit-global
6
+  layeringDefinition:
7
+    abstract: true
8
+    layer: global
9
+  labels:
10
+    hosttype: fluentbit-global
11
+  storagePolicy: cleartext
12
+  substitutions:
13
+    # Chart source
14
+    - src:
15
+        schema: pegleg/SoftwareVersions/v1
16
+        name: software-versions
17
+        path: .charts.osh_infra.fluentbit
18
+      dest:
19
+        path: .source
20
+
21
+    # Images
22
+    - src:
23
+        schema: pegleg/SoftwareVersions/v1
24
+        name: software-versions
25
+        path: .images.osh_infra.fluentbit
26
+      dest:
27
+        path: .values.images.tags
28
+
29
+    # Endpoints
30
+    - src:
31
+        schema: pegleg/EndpointCatalogue/v1
32
+        name: osh_infra_endpoints
33
+        path: .osh_infra.fluentd # TODO change it in OSH repo
34
+      dest:
35
+        path: .values.endpoints.fluentbit
36
+
37
+data:
38
+  chart_name: fluentbit
39
+  release: fluentbit
40
+  namespace: osh-infra
41
+  wait:
42
+    timeout: 900
43
+    labels:
44
+      release_group: airship-fluentbit
45
+  install:
46
+    no_hooks: false
47
+  upgrade:
48
+    no_hooks: false
49
+    pre:
50
+      delete:
51
+        - type: job
52
+          labels:
53
+            release_group: airship-fluentbit
54
+      create: []
55
+    post:
56
+      create: []
57
+  values:
58
+    monitoring:
59
+      prometheus:
60
+        enabled: true
61
+    pod:
62
+      resources:
63
+        enabled: true
64
+        fluentbit:
65
+          limits:
66
+            memory: '4Gi'
67
+            cpu: '2000m'
68
+          requests:
69
+            memory: '2Gi'
70
+            cpu: '1000m'
71
+        jobs:
72
+          image_repo_sync:
73
+            requests:
74
+              memory: '0'
75
+              cpu: '0'
76
+            limits:
77
+              memory: '1024Mi'
78
+              cpu: '2000m'
79
+          tests:
80
+            requests:
81
+              memory: '0'
82
+              cpu: '0'
83
+            limits:
84
+              memory: '1024Mi'
85
+              cpu: '2000m'
86
+    labels:
87
+      fluentbit:
88
+        node_selector_key: fluentbit
89
+        node_selector_value: enabled
90
+      job:
91
+        node_selector_key: openstack-control-plane
92
+        node_selector_value: enabled
93
+    conf:
94
+      fluentbit:
95
+        template: |
96
+          [SERVICE]
97
+              Daemon false
98
+              Flush 5
99
+              Log_Level info
100
+              Parsers_File parsers.conf
101
+
102
+          [INPUT]
103
+              Buffer_Chunk_Size 1M
104
+              Buffer_Max_Size 1M
105
+              Mem_Buf_Limit 5MB
106
+              Name tail
107
+              Path /var/log/kern.log
108
+              Tag kernel
109
+
110
+          [INPUT]
111
+              Buffer_Chunk_Size 1M
112
+              Buffer_Max_Size 1M
113
+              Mem_Buf_Limit 5MB
114
+              Name tail
115
+              Parser docker
116
+              Path /var/log/containers/*.log
117
+              Tag kube.*
118
+
119
+          [INPUT]
120
+              Buffer_Chunk_Size 1M
121
+              Buffer_Max_Size 1M
122
+              Mem_Buf_Limit 5MB
123
+              Name tail
124
+              Path /var/log/libvirt/libvirtd.log
125
+              Tag libvirt
126
+
127
+          [INPUT]
128
+              Buffer_Chunk_Size 1M
129
+              Buffer_Max_Size 1M
130
+              Mem_Buf_Limit 5MB
131
+              Name tail
132
+              Path /var/log/libvirt/qemu/*.log
133
+              Tag qemu
134
+
135
+          [INPUT]
136
+              Buffer_Chunk_Size 1M
137
+              Buffer_Max_Size 1M
138
+              Mem_Buf_Limit 5MB
139
+              Name systemd
140
+              Path ${JOURNAL_PATH}
141
+              Systemd_Filter _SYSTEMD_UNIT=kubelet.service
142
+              Tag journal.*
143
+
144
+          [INPUT]
145
+              Buffer_Chunk_Size 1M
146
+              Buffer_Max_Size 1M
147
+              Mem_Buf_Limit 5MB
148
+              Name systemd
149
+              Path ${JOURNAL_PATH}
150
+              Systemd_Filter _SYSTEMD_UNIT=docker.service
151
+              Tag journal.*
152
+
153
+          [INPUT]
154
+              Buffer_Chunk_Size 1M
155
+              Buffer_Max_Size 1M
156
+              Mem_Buf_Limit 5MB
157
+              Name tail
158
+              Parsers syslog
159
+              Path /var/log/ceph/airship-ucp-ceph-mon/ceph.log
160
+              Tag ceph.cluster.*
161
+
162
+          [INPUT]
163
+              Buffer_Chunk_Size 1M
164
+              Buffer_Max_Size 1M
165
+              Mem_Buf_Limit 5MB
166
+              Name tail
167
+              Parsers syslog
168
+              Path /var/log/ceph/airship-ucp-ceph-mon/ceph.audit.log
169
+              Tag ceph.audit.*
170
+
171
+          [INPUT]
172
+              Buffer_Chunk_Size 1M
173
+              Buffer_Max_Size 1M
174
+              Mem_Buf_Limit 5MB
175
+              Name tail
176
+              Parsers syslog
177
+              Path /var/log/ceph/airship-ucp-ceph-mon/ceph-mon**.log
178
+              Tag ceph.mon.*
179
+
180
+          [INPUT]
181
+              Buffer_Chunk_Size 1M
182
+              Buffer_Max_Size 1M
183
+              Mem_Buf_Limit 5MB
184
+              Name tail
185
+              Parsers syslog
186
+              Path /var/log/ceph/airship-ucp-ceph-osd/ceph-osd**.log
187
+              Tag ceph.osd.*
188
+
189
+          [FILTER]
190
+              Interval 1s
191
+              Match **
192
+              Name throttle
193
+              Rate 1000
194
+              Window 300
195
+
196
+          [FILTER]
197
+              Match libvirt
198
+              Name record_modifier
199
+              Record hostname ${HOSTNAME}
200
+
201
+          [FILTER]
202
+              Match qemu
203
+              Name record_modifier
204
+              Record hostname ${HOSTNAME}
205
+
206
+          [FILTER]
207
+              Match kernel
208
+              Name record_modifier
209
+              Record hostname ${HOSTNAME}
210
+
211
+          [FILTER]
212
+              Match journal.**
213
+              Name modify
214
+              Rename _BOOT_ID BOOT_ID
215
+              Rename _CAP_EFFECTIVE CAP_EFFECTIVE
216
+              Rename _CMDLINE CMDLINE
217
+              Rename _COMM COMM
218
+              Rename _EXE EXE
219
+              Rename _GID GID
220
+              Rename _HOSTNAME HOSTNAME
221
+              Rename _MACHINE_ID MACHINE_ID
222
+              Rename _PID PID
223
+              Rename _SYSTEMD_CGROUP SYSTEMD_CGROUP
224
+              Rename _SYSTEMD_SLICE SYSTEMD_SLICE
225
+              Rename _SYSTEMD_UNIT SYSTEMD_UNIT
226
+              Rename _TRANSPORT TRANSPORT
227
+              Rename _UID UID
228
+
229
+          [OUTPUT]
230
+              Match **.fluentd**
231
+              Name null
232
+
233
+          [FILTER]
234
+              Match kube.*
235
+              Merge_JSON_Log true
236
+              Name kubernetes
237
+
238
+          [OUTPUT]
239
+              Host ${FLUENTD_HOST}
240
+              Match *
241
+              Name forward
242
+              Port ${FLUENTD_PORT}
243
+      parsers:
244
+        template: |
245
+          [PARSER]
246
+            Name syslog
247
+            Format regex
248
+            Regex '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
249
+            Time_Key time
250
+            Time_Format "%Y-%m-%dT%H:%M:%S.%L"
251
+            Time_Keep On
252
+            Types "pid:integer"
253
+  dependencies:
254
+    - osh-infra-helm-toolkit
255
+...

+ 375
- 0
global/software/charts/osh-infra/osh-infra-logging/fluentd.yaml View File

@@ -0,0 +1,375 @@
1
+---
2
+schema: armada/Chart/v1
3
+metadata:
4
+  schema: metadata/Document/v1
5
+  name: fluentd-global
6
+  layeringDefinition:
7
+    abstract: true
8
+    layer: global
9
+  labels:
10
+    hosttype: fluentd-global
11
+  storagePolicy: cleartext
12
+  substitutions:
13
+    # Chart source
14
+    - src:
15
+        schema: pegleg/SoftwareVersions/v1
16
+        name: software-versions
17
+        path: .charts.osh_infra.fluentd
18
+      dest:
19
+        path: .source
20
+
21
+    # Images
22
+    - src:
23
+        schema: pegleg/SoftwareVersions/v1
24
+        name: software-versions
25
+        path: .images.osh_infra.fluentd
26
+      dest:
27
+        path: .values.images.tags
28
+
29
+    # Endpoints
30
+    - src:
31
+        schema: pegleg/EndpointCatalogue/v1
32
+        name: osh_infra_endpoints
33
+        path: .osh_infra.fluentd
34
+      dest:
35
+        path: .values.endpoints.fluentd
36
+    - src:
37
+        schema: pegleg/EndpointCatalogue/v1
38
+        name: osh_infra_endpoints
39
+        path: .osh_infra.prometheus_fluentd_exporter
40
+      dest:
41
+        path: .values.endpoints.prometheus_fluentd_exporter
42
+
43
+    # Accounts
44
+    - src:
45
+        schema: pegleg/AccountCatalogue/v1
46
+        name: osh_infra_service_accounts
47
+        path: .osh_infra.elasticsearch.admin
48
+      dest:
49
+        path: .values.endpoints.elasticsearch.auth.admin
50
+
51
+    # Secrets
52
+    - dest:
53
+        path: .values.endpoints.elasticsearch.auth.admin.password
54
+      src:
55
+        schema: deckhand/Passphrase/v1
56
+        name: osh_infra_elasticsearch_admin_password
57
+        path: .
58
+
59
+data:
60
+  chart_name: fluentd
61
+  release: fluentd
62
+  namespace: osh-infra
63
+  wait:
64
+    timeout: 900
65
+    labels:
66
+      release_group: airship-fluentd
67
+  install:
68
+    no_hooks: false
69
+  upgrade:
70
+    no_hooks: false
71
+    pre:
72
+      delete:
73
+        - type: job
74
+          labels:
75
+            release_group: airship-fluentd
76
+      create: []
77
+    post:
78
+      create: []
79
+  values:
80
+    monitoring:
81
+      prometheus:
82
+        enabled: true
83
+    pod:
84
+      resources:
85
+        enabled: true
86
+        fluentd:
87
+          limits:
88
+            memory: '4Gi'
89
+            cpu: '2000m'
90
+          requests:
91
+            memory: '2Gi'
92
+            cpu: '1000m'
93
+        prometheus_fluentd_exporter:
94
+          limits:
95
+            memory: '1024Mi'
96
+            cpu: '2000m'
97
+          requests:
98
+            memory: '0'
99
+            cpu: '0'
100
+        jobs:
101
+          image_repo_sync:
102
+            requests:
103
+              memory: '0'
104
+              cpu: '0'
105
+            limits:
106
+              memory: '1024Mi'
107
+              cpu: '2000m'
108
+          tests:
109
+            requests:
110
+              memory: '0'
111
+              cpu: '0'
112
+            limits:
113
+              memory: '1024Mi'
114
+              cpu: '2000m'
115
+    labels:
116
+      fluentd:
117
+        node_selector_key: fluentd
118
+        node_selector_value: enabled
119
+      prometheus_fluentd_exporter:
120
+        node_selector_key: openstack-control-plane
121
+        node_selector_value: enabled
122
+      job:
123
+        node_selector_key: openstack-control-plane
124
+        node_selector_value: enabled
125
+    conf:
126
+      fluentd:
127
+        template: |
128
+          <source>
129
+            bind 0.0.0.0
130
+            port 24220
131
+            @type monitor_agent
132
+          </source>
133
+
134
+          <source>
135
+            bind 0.0.0.0
136
+            port "#{ENV['FLUENTD_PORT']}"
137
+            @type forward
138
+          </source>
139
+
140
+          <match fluent.**>
141
+            @type null
142
+          </match>
143
+
144
+          <match kube.var.log.containers.**.log>
145
+            <rule>
146
+              key log
147
+              pattern /info/i
148
+              tag info.${tag}
149
+            </rule>
150
+            <rule>
151
+              key log
152
+              pattern /warn/i
153
+              tag warn.${tag}
154
+            </rule>
155
+            <rule>
156
+              key log
157
+              pattern /error/i
158
+              tag error.${tag}
159
+            </rule>
160
+            <rule>
161
+              key log
162
+              pattern /critical/i
163
+              tag critical.${tag}
164
+            </rule>
165
+            <rule>
166
+              key log
167
+              pattern (.+)
168
+              tag info.${tag}
169
+            </rule>
170
+            @type rewrite_tag_filter
171
+          </match>
172
+
173
+          <filter **.kube.var.log.containers.**.log>
174
+            enable_ruby true
175
+            <record>
176
+              application ${record["kubernetes"]["labels"]["application"]}
177
+              level ${tag_parts[0]}
178
+            </record>
179
+            @type record_transformer
180
+          </filter>
181
+
182
+          <filter openstack.**>
183
+            <record>
184
+              application ${tag_parts[1]}
185
+            </record>
186
+            @type record_transformer
187
+          </filter>
188
+
189
+          <match openstack.**>
190
+            <rule>
191
+              key level
192
+              pattern INFO
193
+              tag info.${tag}
194
+            </rule>
195
+            <rule>
196
+              key level
197
+              pattern WARN
198
+              tag warn.${tag}
199
+            </rule>
200
+            <rule>
201
+              key level
202
+              pattern ERROR
203
+              tag error.${tag}
204
+            </rule>
205
+            <rule>
206
+              key level
207
+              pattern CRITICAL
208
+              tag critical.${tag}
209
+            </rule>
210
+            @type rewrite_tag_filter
211
+          </match>
212
+
213
+          <match *.openstack.**>
214
+            <rule>
215
+              key application
216
+              pattern keystone
217
+              tag auth.${tag}
218
+            </rule>
219
+            <rule>
220
+              key application
221
+              pattern horizon
222
+              tag auth.${tag}
223
+            </rule>
224
+            <rule>
225
+              key application
226
+              pattern mariadb
227
+              tag auth.${tag}
228
+            </rule>
229
+            <rule>
230
+              key application
231
+              pattern memcached
232
+              tag auth.${tag}
233
+            </rule>
234
+            <rule>
235
+              key application
236
+              pattern rabbitmq
237
+              tag auth.${tag}
238
+            </rule>
239
+            @type rewrite_tag_filter
240
+          </match>
241
+
242
+          <match libvirt>
243
+            <buffer>
244
+              chunk_limit_size 8MB
245
+              flush_interval 15s
246
+              flush_thread_count 8
247
+              queue_limit_length 256
248
+              retry_forever false
249
+              retry_max_interval 30
250
+            </buffer>
251
+            host "#{ENV['ELASTICSEARCH_HOST']}"
252
+            reload_connections false
253
+            reconnect_on_error true
254
+            reload_on_failure true
255
+            include_tag_key true
256
+            logstash_format true
257
+            logstash_prefix libvirt
258
+            password "#{ENV['ELASTICSEARCH_PASSWORD']}"
259
+            port "#{ENV['ELASTICSEARCH_PORT']}"
260
+            @type elasticsearch
261
+            user "#{ENV['ELASTICSEARCH_USERNAME']}"
262
+          </match>
263
+
264
+          <match qemu>
265
+            <buffer>
266
+              chunk_limit_size 8MB
267
+              flush_interval 15s
268
+              flush_thread_count 8
269
+              queue_limit_length 256
270
+              retry_forever false
271
+              retry_max_interval 30
272
+            </buffer>
273
+            host "#{ENV['ELASTICSEARCH_HOST']}"
274
+            reload_connections false
275
+            reconnect_on_error true
276
+            reload_on_failure true
277
+            include_tag_key true
278
+            logstash_format true
279
+            logstash_prefix qemu
280
+            password "#{ENV['ELASTICSEARCH_PASSWORD']}"
281
+            port "#{ENV['ELASTICSEARCH_PORT']}"
282
+            @type elasticsearch
283
+            user "#{ENV['ELASTICSEARCH_USERNAME']}"
284
+          </match>
285
+
286
+          <match journal.**>
287
+            <buffer>
288
+              chunk_limit_size 8MB
289
+              flush_interval 15s
290
+              flush_thread_count 8
291
+              queue_limit_length 256
292
+              retry_forever false
293
+              retry_max_interval 30
294
+            </buffer>
295
+            host "#{ENV['ELASTICSEARCH_HOST']}"
296
+            reload_connections false
297
+            reconnect_on_error true
298
+            reload_on_failure true
299
+            include_tag_key true
300
+            logstash_format true
301
+            logstash_prefix journal
302
+            password "#{ENV['ELASTICSEARCH_PASSWORD']}"
303
+            port "#{ENV['ELASTICSEARCH_PORT']}"
304
+            @type elasticsearch
305
+            user "#{ENV['ELASTICSEARCH_USERNAME']}"
306
+          </match>
307
+
308
+          <match kernel>
309
+            <buffer>
310
+              chunk_limit_size 8MB
311
+              flush_interval 15s
312
+              flush_thread_count 8
313
+              queue_limit_length 256
314
+              retry_forever false
315
+              retry_max_interval 30
316
+            </buffer>
317
+            host "#{ENV['ELASTICSEARCH_HOST']}"
318
+            reload_connections false
319
+            reconnect_on_error true
320
+            reload_on_failure true
321
+            include_tag_key true
322
+            logstash_format true
323
+            logstash_prefix kernel
324
+            password "#{ENV['ELASTICSEARCH_PASSWORD']}"
325
+            port "#{ENV['ELASTICSEARCH_PORT']}"
326
+            @type elasticsearch
327
+            user "#{ENV['ELASTICSEARCH_USERNAME']}"
328
+          </match>
329
+
330
+          <match **>
331
+            <buffer>
332
+              chunk_limit_size 8MB
333
+              flush_interval 15s
334
+              flush_thread_count 8
335
+              queue_limit_length 256
336
+              retry_forever false
337
+              retry_max_interval 30
338
+            </buffer>
339
+            host "#{ENV['ELASTICSEARCH_HOST']}"
340
+            reload_connections false
341
+            reconnect_on_error true
342
+            reload_on_failure true
343
+            include_tag_key true
344
+            logstash_format true
345
+            password "#{ENV['ELASTICSEARCH_PASSWORD']}"
346
+            port "#{ENV['ELASTICSEARCH_PORT']}"
347
+            @type elasticsearch
348
+            type_name fluent
349
+            user "#{ENV['ELASTICSEARCH_USERNAME']}"
350
+          </match>
351
+
352
+          <match *ceph-**.log>
353
+            <buffer>
354
+              chunk_limit_size 8MB
355
+              flush_interval 15s
356
+              flush_thread_count 8
357
+              queue_limit_length 256
358
+              retry_forever false
359
+              retry_max_interval 30
360
+            </buffer>
361
+            host "#{ENV['ELASTICSEARCH_HOST']}"
362
+            reload_connections false
363
+            reconnect_on_error true
364
+            reload_on_failure true
365
+            include_tag_key true
366
+            logstash_format true
367
+            logstash_prefix ceph
368
+            password "#{ENV['ELASTICSEARCH_PASSWORD']}"
369
+            port "#{ENV['ELASTICSEARCH_PORT']}"
370
+            @type elasticsearch
371
+            user "#{ENV['ELASTICSEARCH_USERNAME']}"
372
+          </match>
373
+  dependencies:
374
+    - osh-infra-helm-toolkit
375
+...

+ 10
- 4
global/software/config/versions.yaml View File

@@ -260,10 +260,15 @@ data:
260 260
         reference: 1f1482bcb71cd3483c6df3ac4bab09653ed6a2e1
261 261
         subpath: elasticsearch
262 262
         type: git
263
-      fluent_logging:
263
+      fluentbit:
264 264
         location: https://opendev.org/openstack/openstack-helm-infra
265
-        reference: 5e1ecd9840397bf9e8829ce0d98fcb721db1b74e
266
-        subpath: fluent-logging
265
+        reference: 29fc716cf3ff636a66944903fbbe9bd3f921e22f
266
+        subpath: fluentbit
267
+        type: git
268
+      fluentd:
269
+        location: https://opendev.org/openstack/openstack-helm-infra
270
+        reference: 29fc716cf3ff636a66944903fbbe9bd3f921e22f
271
+        subpath: fluentd
267 272
         type: git
268 273
       grafana:
269 274
         location: https://opendev.org/openstack/openstack-helm-infra
@@ -592,7 +597,8 @@ data:
592 597
       tempest: {}
593 598
     osh_infra:
594 599
       elasticsearch: {}
595
-      fluent_logging: {}
600
+      fluentbit: {}
601
+      fluentd: {}
596 602
       grafana: {}
597 603
       kibana: {}
598 604
       nagios: {}

site/airskiff/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml → site/airskiff/software/charts/osh-infra/osh-infra-logging/fluentbit.yaml View File

@@ -4,12 +4,12 @@
4 4
 schema: armada/Chart/v1
5 5
 metadata:
6 6
   schema: metadata/Document/v1
7
-  name: fluent-logging
7
+  name: fluentbit
8 8
   layeringDefinition:
9 9
     abstract: false
10 10
     layer: site
11 11
     parentSelector:
12
-      hosttype: fluent-logging-global
12
+      hosttype: fluentbit-global
13 13
     actions:
14 14
       - method: merge
15 15
         path: .

+ 18
- 0
site/airskiff/software/charts/osh-infra/osh-infra-logging/fluentd.yaml View File

@@ -0,0 +1,18 @@
1
+---
2
+# NOTE: This file is ignored by Airskiff and is copied from the seaworthy site.
3
+
4
+schema: armada/Chart/v1
5
+metadata:
6
+  schema: metadata/Document/v1
7
+  name: fluentd
8
+  layeringDefinition:
9
+    abstract: false
10
+    layer: site
11
+    parentSelector:
12
+      hosttype: fluentd-global
13
+    actions:
14
+      - method: merge
15
+        path: .
16
+  storagePolicy: cleartext
17
+data: {}
18
+...

site/seaworthy/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml → site/seaworthy/software/charts/osh-infra/osh-infra-logging/fluentbit.yaml View File

@@ -2,12 +2,12 @@
2 2
 schema: armada/Chart/v1
3 3
 metadata:
4 4
   schema: metadata/Document/v1
5
-  name: fluent-logging
5
+  name: fluentbit
6 6
   layeringDefinition:
7 7
     abstract: false
8 8
     layer: site
9 9
     parentSelector:
10
-      hosttype: fluent-logging-global
10
+      hosttype: fluentbit-global
11 11
     actions:
12 12
       - method: merge
13 13
         path: .

+ 16
- 0
site/seaworthy/software/charts/osh-infra/osh-infra-logging/fluentd.yaml View File

@@ -0,0 +1,16 @@
1
+---
2
+schema: armada/Chart/v1
3
+metadata:
4
+  schema: metadata/Document/v1
5
+  name: fluentd
6
+  layeringDefinition:
7
+    abstract: false
8
+    layer: site
9
+    parentSelector:
10
+      hosttype: fluentd-global
11
+    actions:
12
+      - method: merge
13
+        path: .
14
+  storagePolicy: cleartext
15
+data: {}
16
+...

type/sloop/charts/osh-infra/fluent-logging.yaml → type/sloop/charts/osh-infra/fluentbit.yaml View File

@@ -2,14 +2,14 @@
2 2
 schema: armada/Chart/v1
3 3
 metadata:
4 4
   schema: metadata/Document/v1
5
-  name: fluent-logging
5
+  name: fluentbit
6 6
   labels:
7
-    name: fluent-logging-type
7
+    name: fluentbit-type
8 8
   layeringDefinition:
9 9
     abstract: false
10 10
     layer: type
11 11
     parentSelector:
12
-      hosttype: fluent-logging-global
12
+      hosttype: fluentbit-global
13 13
     actions:
14 14
       - method: merge
15 15
         path: .

+ 22
- 0
type/sloop/charts/osh-infra/fluentd.yaml View File

@@ -0,0 +1,22 @@
1
+---
2
+schema: armada/Chart/v1
3
+metadata:
4
+  schema: metadata/Document/v1
5
+  name: fluentd
6
+  labels:
7
+    name: fluentd-type
8
+  layeringDefinition:
9
+    abstract: false
10
+    layer: type
11
+    parentSelector:
12
+      hosttype: fluentd-global
13
+    actions:
14
+      - method: merge
15
+        path: .
16
+  storagePolicy: cleartext
17
+data:
18
+  values:
19
+    pod:
20
+      replicas:
21
+        fluentd: 1
22
+...

Loading…
Cancel
Save