Uplift and Separate fluentbit and fluentd charts
fluent-logging chart was splitter into fluentbit and fluentd charts in the following PS: https://review.opendev.org/642146 Change-Id: Ic72f9fb6293b99395a0acf48064b9c04e312d365
This commit is contained in:
Alexander Noskov
parent
0190a2b5b0
commit
87b26958a4
@ -415,7 +415,9 @@ Under site/airsloop/software/config/ create a versions.yaml file in the followin
|
||||
osh_infra:
|
||||
elasticsearch:
|
||||
proxy_server: proxy.example.com:8080
|
||||
fluent_logging:
|
||||
fluentbit:
|
||||
proxy_server: proxy.example.com:8080
|
||||
fluentd:
|
||||
proxy_server: proxy.example.com:8080
|
||||
grafana:
|
||||
proxy_server: proxy.example.com:8080
|
||||
|
||||
@ -235,7 +235,18 @@ data:
|
||||
type: string
|
||||
reference:
|
||||
type: string
|
||||
fluent_logging:
|
||||
fluentbit:
|
||||
type: object
|
||||
properties:
|
||||
type:
|
||||
type: string
|
||||
location:
|
||||
type: string
|
||||
subpath:
|
||||
type: string
|
||||
reference:
|
||||
type: string
|
||||
fluentd:
|
||||
type: object
|
||||
properties:
|
||||
type:
|
||||
|
||||
@ -12,4 +12,5 @@ data:
|
||||
sequenced: True
|
||||
chart_group:
|
||||
- elasticsearch
|
||||
- fluent-logging
|
||||
- fluentbit
|
||||
- fluentd
|
||||
|
||||
@ -1,419 +0,0 @@
|
||||
---
|
||||
schema: armada/Chart/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: fluent-logging-global
|
||||
layeringDefinition:
|
||||
abstract: true
|
||||
layer: global
|
||||
labels:
|
||||
hosttype: fluent-logging-global
|
||||
storagePolicy: cleartext
|
||||
substitutions:
|
||||
# Chart source
|
||||
- src:
|
||||
schema: pegleg/SoftwareVersions/v1
|
||||
name: software-versions
|
||||
path: .charts.osh_infra.fluent_logging
|
||||
dest:
|
||||
path: .source
|
||||
|
||||
# Images
|
||||
- src:
|
||||
schema: pegleg/SoftwareVersions/v1
|
||||
name: software-versions
|
||||
path: .images.osh_infra.fluent_logging
|
||||
dest:
|
||||
path: .values.images.tags
|
||||
|
||||
# Endpoints
|
||||
- src:
|
||||
schema: pegleg/EndpointCatalogue/v1
|
||||
name: osh_infra_endpoints
|
||||
path: .osh_infra.elasticsearch
|
||||
dest:
|
||||
path: .values.endpoints.elasticsearch
|
||||
- src:
|
||||
schema: pegleg/EndpointCatalogue/v1
|
||||
name: osh_infra_endpoints
|
||||
path: .osh_infra.fluentd
|
||||
dest:
|
||||
path: .values.endpoints.fluentd
|
||||
- src:
|
||||
schema: pegleg/EndpointCatalogue/v1
|
||||
name: osh_infra_endpoints
|
||||
path: .osh_infra.prometheus_fluentd_exporter
|
||||
dest:
|
||||
path: .values.endpoints.prometheus_fluentd_exporter
|
||||
# Accounts
|
||||
- src:
|
||||
schema: pegleg/AccountCatalogue/v1
|
||||
name: osh_infra_service_accounts
|
||||
path: .osh_infra.elasticsearch.admin
|
||||
dest:
|
||||
path: .values.endpoints.elasticsearch.auth.admin
|
||||
|
||||
# Secrets
|
||||
- dest:
|
||||
path: .values.endpoints.elasticsearch.auth.admin.password
|
||||
src:
|
||||
schema: deckhand/Passphrase/v1
|
||||
name: osh_infra_elasticsearch_admin_password
|
||||
path: .
|
||||
|
||||
data:
|
||||
chart_name: fluent-logging
|
||||
release: fluent-logging
|
||||
namespace: osh-infra
|
||||
wait:
|
||||
timeout: 900
|
||||
labels:
|
||||
release_group: airship-fluent-logging
|
||||
install:
|
||||
no_hooks: false
|
||||
upgrade:
|
||||
no_hooks: false
|
||||
pre:
|
||||
delete:
|
||||
- type: job
|
||||
labels:
|
||||
release_group: airship-fluent-logging
|
||||
create: []
|
||||
post:
|
||||
create: []
|
||||
values:
|
||||
monitoring:
|
||||
prometheus:
|
||||
enabled: true
|
||||
pod:
|
||||
resources:
|
||||
enabled: true
|
||||
fluentbit:
|
||||
limits:
|
||||
memory: '4Gi'
|
||||
cpu: '2000m'
|
||||
requests:
|
||||
memory: '2Gi'
|
||||
cpu: '1000m'
|
||||
fluentd:
|
||||
limits:
|
||||
memory: '4Gi'
|
||||
cpu: '2000m'
|
||||
requests:
|
||||
memory: '2Gi'
|
||||
cpu: '1000m'
|
||||
prometheus_fluentd_exporter:
|
||||
limits:
|
||||
memory: '1024Mi'
|
||||
cpu: '2000m'
|
||||
requests:
|
||||
memory: '0'
|
||||
cpu: '0'
|
||||
jobs:
|
||||
image_repo_sync:
|
||||
requests:
|
||||
memory: '0'
|
||||
cpu: '0'
|
||||
limits:
|
||||
memory: '1024Mi'
|
||||
cpu: '2000m'
|
||||
tests:
|
||||
requests:
|
||||
memory: '0'
|
||||
cpu: '0'
|
||||
limits:
|
||||
memory: '1024Mi'
|
||||
cpu: '2000m'
|
||||
labels:
|
||||
fluentd:
|
||||
node_selector_key: fluentd
|
||||
node_selector_value: enabled
|
||||
fluentbit:
|
||||
node_selector_key: fluentbit
|
||||
node_selector_value: enabled
|
||||
prometheus_fluentd_exporter:
|
||||
node_selector_key: openstack-control-plane
|
||||
node_selector_value: enabled
|
||||
job:
|
||||
node_selector_key: openstack-control-plane
|
||||
node_selector_value: enabled
|
||||
conf:
|
||||
fluentbit:
|
||||
- service:
|
||||
header: service
|
||||
Flush: 5
|
||||
Daemon: Off
|
||||
Log_Level: info
|
||||
Parsers_File: parsers.conf
|
||||
- kernel_messages:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: kernel
|
||||
Path: /var/log/kern.log
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- kubelet:
|
||||
header: input
|
||||
Name: systemd
|
||||
Tag: journal.*
|
||||
Path: ${JOURNAL_PATH}
|
||||
Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- docker_daemon:
|
||||
header: input
|
||||
Name: systemd
|
||||
Tag: journal.*
|
||||
Path: ${JOURNAL_PATH}
|
||||
Systemd_Filter: _SYSTEMD_UNIT=docker.service
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- libvirt:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: libvirt
|
||||
Path: /var/log/libvirt/libvirtd.log
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- qemu:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: qemu
|
||||
Path: /var/log/libvirt/qemu/*.log
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- libvirt_record_modifier:
|
||||
header: filter
|
||||
Name: record_modifier
|
||||
Match: libvirt
|
||||
Record: hostname ${HOSTNAME}
|
||||
- qemu_record_modifier:
|
||||
header: filter
|
||||
Name: record_modifier
|
||||
Match: qemu
|
||||
Record: hostname ${HOSTNAME}
|
||||
- kernel_record_modifier:
|
||||
header: filter
|
||||
Name: record_modifier
|
||||
Match: kernel
|
||||
Record: hostname ${HOSTNAME}
|
||||
- systemd_modify_fields:
|
||||
header: filter
|
||||
Name: modify
|
||||
Match: journal.**
|
||||
Rename:
|
||||
_BOOT_ID: BOOT_ID
|
||||
_CAP_EFFECTIVE: CAP_EFFECTIVE
|
||||
_CMDLINE: CMDLINE
|
||||
_COMM: COMM
|
||||
_EXE: EXE
|
||||
_GID: GID
|
||||
_HOSTNAME: HOSTNAME
|
||||
_MACHINE_ID: MACHINE_ID
|
||||
_PID: PID
|
||||
_SYSTEMD_CGROUP: SYSTEMD_CGROUP
|
||||
_SYSTEMD_SLICE: SYSTEMD_SLICE
|
||||
_SYSTEMD_UNIT: SYSTEMD_UNIT
|
||||
_UID: UID
|
||||
_TRANSPORT: TRANSPORT
|
||||
- ceph_cluster_logs:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: ceph.cluster.*
|
||||
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.log
|
||||
Parsers: syslog
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- ceph_audit_logs:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: ceph.audit.*
|
||||
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.audit.log
|
||||
Parsers: syslog
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- ceph_mon_logs:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: ceph.mon.*
|
||||
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph-mon**.log
|
||||
Parsers: syslog
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- ceph_osd_logs:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: ceph.osd.*
|
||||
Path: /var/log/ceph/airship-ucp-ceph-osd/ceph-osd**.log
|
||||
Parsers: syslog
|
||||
Mem_Buf_Limit: 5MB
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
- containers_tail:
|
||||
header: input
|
||||
Name: tail
|
||||
Tag: kube.*
|
||||
Path: /var/log/containers/*.log
|
||||
Parser: docker
|
||||
Buffer_Chunk_Size: 1M
|
||||
Buffer_Max_Size: 1M
|
||||
Mem_Buf_Limit: 5MB
|
||||
- drop_fluentd_logs:
|
||||
header: output
|
||||
Name: "null"
|
||||
Match: "**.fluentd**"
|
||||
- kube_filter:
|
||||
header: filter
|
||||
Name: kubernetes
|
||||
Match: kube.*
|
||||
Merge_JSON_Log: On
|
||||
- throttle_filter:
|
||||
header: filter
|
||||
Name: throttle
|
||||
Match: "**"
|
||||
Rate: 1000
|
||||
Window: 300
|
||||
Interval: 1s
|
||||
- fluentd_output:
|
||||
header: output
|
||||
Name: forward
|
||||
Match: "*"
|
||||
Host: ${FLUENTD_HOST}
|
||||
Port: ${FLUENTD_PORT}
|
||||
fluentd:
|
||||
- metrics_agent:
|
||||
header: source
|
||||
type: monitor_agent
|
||||
bind: 0.0.0.0
|
||||
port: 24220
|
||||
- fluentbit_forward:
|
||||
header: source
|
||||
type: forward
|
||||
port: "#{ENV['FLUENTD_PORT']}"
|
||||
bind: 0.0.0.0
|
||||
- filter_fluentd_logs:
|
||||
header: match
|
||||
expression: "fluent.**"
|
||||
type: "null"
|
||||
- journal_elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "journal.**"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
logstash_prefix: journal
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
- kernel_elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "kernel"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
logstash_prefix: kernel
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
- ceph_elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "**ceph-**.log"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
logstash_prefix: ceph
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
- oslo_fluentd_elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "**.openstack.*"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
logstash_prefix: openstack
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
type_name: oslo_openstack_fluentd
|
||||
- elasticsearch:
|
||||
header: match
|
||||
type: elasticsearch
|
||||
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
expression: "**"
|
||||
include_tag_key: true
|
||||
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
logstash_format: true
|
||||
buffer:
|
||||
-
|
||||
- header: buffer
|
||||
flush_thread_count: 8
|
||||
flush_interval: "15"
|
||||
chunk_limit_size: 8MB
|
||||
queue_limit_length: 256
|
||||
retry_max_interval: 30
|
||||
retry_forever: false
|
||||
flush_interval: "10"
|
||||
parsers:
|
||||
- syslog:
|
||||
header: parser
|
||||
Name: syslog
|
||||
Format: regex
|
||||
Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
|
||||
Time_Key: time
|
||||
Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
|
||||
Time_Keep: On
|
||||
Types: "pid:integer"
|
||||
dependencies:
|
||||
- osh-infra-helm-toolkit
|
||||
...
|
||||
@ -0,0 +1,255 @@
|
||||
---
|
||||
schema: armada/Chart/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: fluentbit-global
|
||||
layeringDefinition:
|
||||
abstract: true
|
||||
layer: global
|
||||
labels:
|
||||
hosttype: fluentbit-global
|
||||
storagePolicy: cleartext
|
||||
substitutions:
|
||||
# Chart source
|
||||
- src:
|
||||
schema: pegleg/SoftwareVersions/v1
|
||||
name: software-versions
|
||||
path: .charts.osh_infra.fluentbit
|
||||
dest:
|
||||
path: .source
|
||||
|
||||
# Images
|
||||
- src:
|
||||
schema: pegleg/SoftwareVersions/v1
|
||||
name: software-versions
|
||||
path: .images.osh_infra.fluentbit
|
||||
dest:
|
||||
path: .values.images.tags
|
||||
|
||||
# Endpoints
|
||||
- src:
|
||||
schema: pegleg/EndpointCatalogue/v1
|
||||
name: osh_infra_endpoints
|
||||
path: .osh_infra.fluentd # TODO change it in OSH repo
|
||||
dest:
|
||||
path: .values.endpoints.fluentbit
|
||||
|
||||
data:
|
||||
chart_name: fluentbit
|
||||
release: fluentbit
|
||||
namespace: osh-infra
|
||||
wait:
|
||||
timeout: 900
|
||||
labels:
|
||||
release_group: airship-fluentbit
|
||||
install:
|
||||
no_hooks: false
|
||||
upgrade:
|
||||
no_hooks: false
|
||||
pre:
|
||||
delete:
|
||||
- type: job
|
||||
labels:
|
||||
release_group: airship-fluentbit
|
||||
create: []
|
||||
post:
|
||||
create: []
|
||||
values:
|
||||
monitoring:
|
||||
prometheus:
|
||||
enabled: true
|
||||
pod:
|
||||
resources:
|
||||
enabled: true
|
||||
fluentbit:
|
||||
limits:
|
||||
memory: '4Gi'
|
||||
cpu: '2000m'
|
||||
requests:
|
||||
memory: '2Gi'
|
||||
cpu: '1000m'
|
||||
jobs:
|
||||
image_repo_sync:
|
||||
requests:
|
||||
memory: '0'
|
||||
cpu: '0'
|
||||
limits:
|
||||
memory: '1024Mi'
|
||||
cpu: '2000m'
|
||||
tests:
|
||||
requests:
|
||||
memory: '0'
|
||||
cpu: '0'
|
||||
limits:
|
||||
memory: '1024Mi'
|
||||
cpu: '2000m'
|
||||
labels:
|
||||
fluentbit:
|
||||
node_selector_key: fluentbit
|
||||
node_selector_value: enabled
|
||||
job:
|
||||
node_selector_key: openstack-control-plane
|
||||
node_selector_value: enabled
|
||||
conf:
|
||||
fluentbit:
|
||||
template: |
|
||||
[SERVICE]
|
||||
Daemon false
|
||||
Flush 5
|
||||
Log_Level info
|
||||
Parsers_File parsers.conf
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name tail
|
||||
Path /var/log/kern.log
|
||||
Tag kernel
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name tail
|
||||
Parser docker
|
||||
Path /var/log/containers/*.log
|
||||
Tag kube.*
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name tail
|
||||
Path /var/log/libvirt/libvirtd.log
|
||||
Tag libvirt
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name tail
|
||||
Path /var/log/libvirt/qemu/*.log
|
||||
Tag qemu
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name systemd
|
||||
Path ${JOURNAL_PATH}
|
||||
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
|
||||
Tag journal.*
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name systemd
|
||||
Path ${JOURNAL_PATH}
|
||||
Systemd_Filter _SYSTEMD_UNIT=docker.service
|
||||
Tag journal.*
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name tail
|
||||
Parsers syslog
|
||||
Path /var/log/ceph/airship-ucp-ceph-mon/ceph.log
|
||||
Tag ceph.cluster.*
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name tail
|
||||
Parsers syslog
|
||||
Path /var/log/ceph/airship-ucp-ceph-mon/ceph.audit.log
|
||||
Tag ceph.audit.*
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name tail
|
||||
Parsers syslog
|
||||
Path /var/log/ceph/airship-ucp-ceph-mon/ceph-mon**.log
|
||||
Tag ceph.mon.*
|
||||
|
||||
[INPUT]
|
||||
Buffer_Chunk_Size 1M
|
||||
Buffer_Max_Size 1M
|
||||
Mem_Buf_Limit 5MB
|
||||
Name tail
|
||||
Parsers syslog
|
||||
Path /var/log/ceph/airship-ucp-ceph-osd/ceph-osd**.log
|
||||
Tag ceph.osd.*
|
||||
|
||||
[FILTER]
|
||||
Interval 1s
|
||||
Match **
|
||||
Name throttle
|
||||
Rate 1000
|
||||
Window 300
|
||||
|
||||
[FILTER]
|
||||
Match libvirt
|
||||
Name record_modifier
|
||||
Record hostname ${HOSTNAME}
|
||||
|
||||
[FILTER]
|
||||
Match qemu
|
||||
Name record_modifier
|
||||
Record hostname ${HOSTNAME}
|
||||
|
||||
[FILTER]
|
||||
Match kernel
|
||||
Name record_modifier
|
||||
Record hostname ${HOSTNAME}
|
||||
|
||||
[FILTER]
|
||||
Match journal.**
|
||||
Name modify
|
||||
Rename _BOOT_ID BOOT_ID
|
||||
Rename _CAP_EFFECTIVE CAP_EFFECTIVE
|
||||
Rename _CMDLINE CMDLINE
|
||||
Rename _COMM COMM
|
||||
Rename _EXE EXE
|
||||
Rename _GID GID
|
||||
Rename _HOSTNAME HOSTNAME
|
||||
Rename _MACHINE_ID MACHINE_ID
|
||||
Rename _PID PID
|
||||
Rename _SYSTEMD_CGROUP SYSTEMD_CGROUP
|
||||
Rename _SYSTEMD_SLICE SYSTEMD_SLICE
|
||||
Rename _SYSTEMD_UNIT SYSTEMD_UNIT
|
||||
Rename _TRANSPORT TRANSPORT
|
||||
Rename _UID UID
|
||||
|
||||
[OUTPUT]
|
||||
Match **.fluentd**
|
||||
Name null
|
||||
|
||||
[FILTER]
|
||||
Match kube.*
|
||||
Merge_JSON_Log true
|
||||
Name kubernetes
|
||||
|
||||
[OUTPUT]
|
||||
Host ${FLUENTD_HOST}
|
||||
Match *
|
||||
Name forward
|
||||
Port ${FLUENTD_PORT}
|
||||
parsers:
|
||||
template: |
|
||||
[PARSER]
|
||||
Name syslog
|
||||
Format regex
|
||||
Regex '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
|
||||
Time_Key time
|
||||
Time_Format "%Y-%m-%dT%H:%M:%S.%L"
|
||||
Time_Keep On
|
||||
Types "pid:integer"
|
||||
dependencies:
|
||||
- osh-infra-helm-toolkit
|
||||
...
|
||||
375
global/software/charts/osh-infra/osh-infra-logging/fluentd.yaml
Normal file
375
global/software/charts/osh-infra/osh-infra-logging/fluentd.yaml
Normal file
@ -0,0 +1,375 @@
|
||||
---
|
||||
schema: armada/Chart/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: fluentd-global
|
||||
layeringDefinition:
|
||||
abstract: true
|
||||
layer: global
|
||||
labels:
|
||||
hosttype: fluentd-global
|
||||
storagePolicy: cleartext
|
||||
substitutions:
|
||||
# Chart source
|
||||
- src:
|
||||
schema: pegleg/SoftwareVersions/v1
|
||||
name: software-versions
|
||||
path: .charts.osh_infra.fluentd
|
||||
dest:
|
||||
path: .source
|
||||
|
||||
# Images
|
||||
- src:
|
||||
schema: pegleg/SoftwareVersions/v1
|
||||
name: software-versions
|
||||
path: .images.osh_infra.fluentd
|
||||
dest:
|
||||
path: .values.images.tags
|
||||
|
||||
# Endpoints
|
||||
- src:
|
||||
schema: pegleg/EndpointCatalogue/v1
|
||||
name: osh_infra_endpoints
|
||||
path: .osh_infra.fluentd
|
||||
dest:
|
||||
path: .values.endpoints.fluentd
|
||||
- src:
|
||||
schema: pegleg/EndpointCatalogue/v1
|
||||
name: osh_infra_endpoints
|
||||
path: .osh_infra.prometheus_fluentd_exporter
|
||||
dest:
|
||||
path: .values.endpoints.prometheus_fluentd_exporter
|
||||
|
||||
# Accounts
|
||||
- src:
|
||||
schema: pegleg/AccountCatalogue/v1
|
||||
name: osh_infra_service_accounts
|
||||
path: .osh_infra.elasticsearch.admin
|
||||
dest:
|
||||
path: .values.endpoints.elasticsearch.auth.admin
|
||||
|
||||
# Secrets
|
||||
- dest:
|
||||
path: .values.endpoints.elasticsearch.auth.admin.password
|
||||
src:
|
||||
schema: deckhand/Passphrase/v1
|
||||
name: osh_infra_elasticsearch_admin_password
|
||||
path: .
|
||||
|
||||
data:
|
||||
chart_name: fluentd
|
||||
release: fluentd
|
||||
namespace: osh-infra
|
||||
wait:
|
||||
timeout: 900
|
||||
labels:
|
||||
release_group: airship-fluentd
|
||||
install:
|
||||
no_hooks: false
|
||||
upgrade:
|
||||
no_hooks: false
|
||||
pre:
|
||||
delete:
|
||||
- type: job
|
||||
labels:
|
||||
release_group: airship-fluentd
|
||||
create: []
|
||||
post:
|
||||
create: []
|
||||
values:
|
||||
monitoring:
|
||||
prometheus:
|
||||
enabled: true
|
||||
pod:
|
||||
resources:
|
||||
enabled: true
|
||||
fluentd:
|
||||
limits:
|
||||
memory: '4Gi'
|
||||
cpu: '2000m'
|
||||
requests:
|
||||
memory: '2Gi'
|
||||
cpu: '1000m'
|
||||
prometheus_fluentd_exporter:
|
||||
limits:
|
||||
memory: '1024Mi'
|
||||
cpu: '2000m'
|
||||
requests:
|
||||
memory: '0'
|
||||
cpu: '0'
|
||||
jobs:
|
||||
image_repo_sync:
|
||||
requests:
|
||||
memory: '0'
|
||||
cpu: '0'
|
||||
limits:
|
||||
memory: '1024Mi'
|
||||
cpu: '2000m'
|
||||
tests:
|
||||
requests:
|
||||
memory: '0'
|
||||
cpu: '0'
|
||||
limits:
|
||||
memory: '1024Mi'
|
||||
cpu: '2000m'
|
||||
labels:
|
||||
fluentd:
|
||||
node_selector_key: fluentd
|
||||
node_selector_value: enabled
|
||||
prometheus_fluentd_exporter:
|
||||
node_selector_key: openstack-control-plane
|
||||
node_selector_value: enabled
|
||||
job:
|
||||
node_selector_key: openstack-control-plane
|
||||
node_selector_value: enabled
|
||||
conf:
|
||||
fluentd:
|
||||
template: |
|
||||
<source>
|
||||
bind 0.0.0.0
|
||||
port 24220
|
||||
@type monitor_agent
|
||||
</source>
|
||||
|
||||
<source>
|
||||
bind 0.0.0.0
|
||||
port "#{ENV['FLUENTD_PORT']}"
|
||||
@type forward
|
||||
</source>
|
||||
|
||||
<match fluent.**>
|
||||
@type null
|
||||
</match>
|
||||
|
||||
<match kube.var.log.containers.**.log>
|
||||
<rule>
|
||||
key log
|
||||
pattern /info/i
|
||||
tag info.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key log
|
||||
pattern /warn/i
|
||||
tag warn.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key log
|
||||
pattern /error/i
|
||||
tag error.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key log
|
||||
pattern /critical/i
|
||||
tag critical.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key log
|
||||
pattern (.+)
|
||||
tag info.${tag}
|
||||
</rule>
|
||||
@type rewrite_tag_filter
|
||||
</match>
|
||||
|
||||
<filter **.kube.var.log.containers.**.log>
|
||||
enable_ruby true
|
||||
<record>
|
||||
application ${record["kubernetes"]["labels"]["application"]}
|
||||
level ${tag_parts[0]}
|
||||
</record>
|
||||
@type record_transformer
|
||||
</filter>
|
||||
|
||||
<filter openstack.**>
|
||||
<record>
|
||||
application ${tag_parts[1]}
|
||||
</record>
|
||||
@type record_transformer
|
||||
</filter>
|
||||
|
||||
<match openstack.**>
|
||||
<rule>
|
||||
key level
|
||||
pattern INFO
|
||||
tag info.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key level
|
||||
pattern WARN
|
||||
tag warn.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key level
|
||||
pattern ERROR
|
||||
tag error.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key level
|
||||
pattern CRITICAL
|
||||
tag critical.${tag}
|
||||
</rule>
|
||||
@type rewrite_tag_filter
|
||||
</match>
|
||||
|
||||
<match *.openstack.**>
|
||||
<rule>
|
||||
key application
|
||||
pattern keystone
|
||||
tag auth.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key application
|
||||
pattern horizon
|
||||
tag auth.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key application
|
||||
pattern mariadb
|
||||
tag auth.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key application
|
||||
pattern memcached
|
||||
tag auth.${tag}
|
||||
</rule>
|
||||
<rule>
|
||||
key application
|
||||
pattern rabbitmq
|
||||
tag auth.${tag}
|
||||
</rule>
|
||||
@type rewrite_tag_filter
|
||||
</match>
|
||||
|
||||
<match libvirt>
|
||||
<buffer>
|
||||
chunk_limit_size 8MB
|
||||
flush_interval 15s
|
||||
flush_thread_count 8
|
||||
queue_limit_length 256
|
||||
retry_forever false
|
||||
retry_max_interval 30
|
||||
</buffer>
|
||||
host "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
reload_connections false
|
||||
reconnect_on_error true
|
||||
reload_on_failure true
|
||||
include_tag_key true
|
||||
logstash_format true
|
||||
logstash_prefix libvirt
|
||||
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
port "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
@type elasticsearch
|
||||
user "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
</match>
|
||||
|
||||
<match qemu>
|
||||
<buffer>
|
||||
chunk_limit_size 8MB
|
||||
flush_interval 15s
|
||||
flush_thread_count 8
|
||||
queue_limit_length 256
|
||||
retry_forever false
|
||||
retry_max_interval 30
|
||||
</buffer>
|
||||
host "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
reload_connections false
|
||||
reconnect_on_error true
|
||||
reload_on_failure true
|
||||
include_tag_key true
|
||||
logstash_format true
|
||||
logstash_prefix qemu
|
||||
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
port "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
@type elasticsearch
|
||||
user "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
</match>
|
||||
|
||||
<match journal.**>
|
||||
<buffer>
|
||||
chunk_limit_size 8MB
|
||||
flush_interval 15s
|
||||
flush_thread_count 8
|
||||
queue_limit_length 256
|
||||
retry_forever false
|
||||
retry_max_interval 30
|
||||
</buffer>
|
||||
host "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
reload_connections false
|
||||
reconnect_on_error true
|
||||
reload_on_failure true
|
||||
include_tag_key true
|
||||
logstash_format true
|
||||
logstash_prefix journal
|
||||
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
port "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
@type elasticsearch
|
||||
user "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
</match>
|
||||
|
||||
<match kernel>
|
||||
<buffer>
|
||||
chunk_limit_size 8MB
|
||||
flush_interval 15s
|
||||
flush_thread_count 8
|
||||
queue_limit_length 256
|
||||
retry_forever false
|
||||
retry_max_interval 30
|
||||
</buffer>
|
||||
host "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
reload_connections false
|
||||
reconnect_on_error true
|
||||
reload_on_failure true
|
||||
include_tag_key true
|
||||
logstash_format true
|
||||
logstash_prefix kernel
|
||||
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
port "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
@type elasticsearch
|
||||
user "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
</match>
|
||||
|
||||
<match **>
|
||||
<buffer>
|
||||
chunk_limit_size 8MB
|
||||
flush_interval 15s
|
||||
flush_thread_count 8
|
||||
queue_limit_length 256
|
||||
retry_forever false
|
||||
retry_max_interval 30
|
||||
</buffer>
|
||||
host "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
reload_connections false
|
||||
reconnect_on_error true
|
||||
reload_on_failure true
|
||||
include_tag_key true
|
||||
logstash_format true
|
||||
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
port "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
@type elasticsearch
|
||||
type_name fluent
|
||||
user "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
</match>
|
||||
|
||||
<match *ceph-**.log>
|
||||
<buffer>
|
||||
chunk_limit_size 8MB
|
||||
flush_interval 15s
|
||||
flush_thread_count 8
|
||||
queue_limit_length 256
|
||||
retry_forever false
|
||||
retry_max_interval 30
|
||||
</buffer>
|
||||
host "#{ENV['ELASTICSEARCH_HOST']}"
|
||||
reload_connections false
|
||||
reconnect_on_error true
|
||||
reload_on_failure true
|
||||
include_tag_key true
|
||||
logstash_format true
|
||||
logstash_prefix ceph
|
||||
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
||||
port "#{ENV['ELASTICSEARCH_PORT']}"
|
||||
@type elasticsearch
|
||||
user "#{ENV['ELASTICSEARCH_USERNAME']}"
|
||||
</match>
|
||||
dependencies:
|
||||
- osh-infra-helm-toolkit
|
||||
...
|
||||
@ -260,10 +260,15 @@ data:
|
||||
reference: 1f1482bcb71cd3483c6df3ac4bab09653ed6a2e1
|
||||
subpath: elasticsearch
|
||||
type: git
|
||||
fluent_logging:
|
||||
fluentbit:
|
||||
location: https://opendev.org/openstack/openstack-helm-infra
|
||||
reference: 5e1ecd9840397bf9e8829ce0d98fcb721db1b74e
|
||||
subpath: fluent-logging
|
||||
reference: 29fc716cf3ff636a66944903fbbe9bd3f921e22f
|
||||
subpath: fluentbit
|
||||
type: git
|
||||
fluentd:
|
||||
location: https://opendev.org/openstack/openstack-helm-infra
|
||||
reference: 29fc716cf3ff636a66944903fbbe9bd3f921e22f
|
||||
subpath: fluentd
|
||||
type: git
|
||||
grafana:
|
||||
location: https://opendev.org/openstack/openstack-helm-infra
|
||||
@ -592,7 +597,8 @@ data:
|
||||
tempest: {}
|
||||
osh_infra:
|
||||
elasticsearch: {}
|
||||
fluent_logging: {}
|
||||
fluentbit: {}
|
||||
fluentd: {}
|
||||
grafana: {}
|
||||
kibana: {}
|
||||
nagios: {}
|
||||
|
||||
@ -4,12 +4,12 @@
|
||||
schema: armada/Chart/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: fluent-logging
|
||||
name: fluentbit
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: site
|
||||
parentSelector:
|
||||
hosttype: fluent-logging-global
|
||||
hosttype: fluentbit-global
|
||||
actions:
|
||||
- method: merge
|
||||
path: .
|
||||
@ -0,0 +1,18 @@
|
||||
---
|
||||
# NOTE: This file is ignored by Airskiff and is copied from the seaworthy site.
|
||||
|
||||
schema: armada/Chart/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: fluentd
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: site
|
||||
parentSelector:
|
||||
hosttype: fluentd-global
|
||||
actions:
|
||||
- method: merge
|
||||
path: .
|
||||
storagePolicy: cleartext
|
||||
data: {}
|
||||
...
|
||||
@ -2,12 +2,12 @@
|
||||
schema: armada/Chart/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: fluent-logging
|
||||
name: fluentbit
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: site
|
||||
parentSelector:
|
||||
hosttype: fluent-logging-global
|
||||
hosttype: fluentbit-global
|
||||
actions:
|
||||
- method: merge
|
||||
path: .
|
||||
@ -0,0 +1,16 @@
|
||||
---
|
||||
schema: armada/Chart/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: fluentd
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: site
|
||||
parentSelector:
|
||||
hosttype: fluentd-global
|
||||
actions:
|
||||
- method: merge
|
||||
path: .
|
||||
storagePolicy: cleartext
|
||||
data: {}
|
||||
...
|
||||
@ -2,14 +2,14 @@
|
||||
schema: armada/Chart/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: fluent-logging
|
||||
name: fluentbit
|
||||
labels:
|
||||
name: fluent-logging-type
|
||||
name: fluentbit-type
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: type
|
||||
parentSelector:
|
||||
hosttype: fluent-logging-global
|
||||
hosttype: fluentbit-global
|
||||
actions:
|
||||
- method: merge
|
||||
path: .
|
||||
22
type/sloop/charts/osh-infra/fluentd.yaml
Normal file
22
type/sloop/charts/osh-infra/fluentd.yaml
Normal file
@ -0,0 +1,22 @@
|
||||
---
|
||||
schema: armada/Chart/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: fluentd
|
||||
labels:
|
||||
name: fluentd-type
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: type
|
||||
parentSelector:
|
||||
hosttype: fluentd-global
|
||||
actions:
|
||||
- method: merge
|
||||
path: .
|
||||
storagePolicy: cleartext
|
||||
data:
|
||||
values:
|
||||
pod:
|
||||
replicas:
|
||||
fluentd: 1
|
||||
...
|
||||
Loading…
Reference in New Issue
Block a user