treasuremap/global/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml

433 lines
13 KiB
YAML

---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: fluent-logging-global
layeringDefinition:
abstract: true
layer: global
labels:
hosttype: fluent-logging-global
storagePolicy: cleartext
substitutions:
# Chart source
- src:
schema: pegleg/SoftwareVersions/v1
name: software-versions
path: .charts.osh_infra.fluent_logging
dest:
path: .source
# Images
- src:
schema: pegleg/SoftwareVersions/v1
name: software-versions
path: .images.osh_infra.fluent_logging
dest:
path: .values.images.tags
# Endpoints
- src:
schema: pegleg/EndpointCatalogue/v1
name: osh_infra_endpoints
path: .osh_infra.elasticsearch
dest:
path: .values.endpoints.elasticsearch
- src:
schema: pegleg/EndpointCatalogue/v1
name: osh_infra_endpoints
path: .osh_infra.fluentd
dest:
path: .values.endpoints.fluentd
- src:
schema: pegleg/EndpointCatalogue/v1
name: osh_infra_endpoints
path: .osh_infra.prometheus_fluentd_exporter
dest:
path: .values.endpoints.prometheus_fluentd_exporter
# Accounts
- src:
schema: pegleg/AccountCatalogue/v1
name: osh_infra_service_accounts
path: .osh_infra.elasticsearch.admin
dest:
path: .values.endpoints.elasticsearch.auth.admin
# Secrets
- dest:
path: .values.endpoints.elasticsearch.auth.admin.password
src:
schema: deckhand/Passphrase/v1
name: osh_infra_elasticsearch_admin_password
path: .
data:
chart_name: fluent-logging
release: fluent-logging
namespace: osh-infra
wait:
timeout: 900
labels:
release_group: airship-fluent-logging
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete:
- type: job
labels:
release_group: airship-fluent-logging
create: []
post:
create: []
values:
monitoring:
prometheus:
enabled: true
pod:
resources:
enabled: true
fluentbit:
limits:
memory: '4Gi'
cpu: '2000m'
requests:
memory: '2Gi'
cpu: '1000m'
fluentd:
limits:
memory: '4Gi'
cpu: '2000m'
requests:
memory: '2Gi'
cpu: '1000m'
prometheus_fluentd_exporter:
limits:
memory: '1024Mi'
cpu: '2000m'
requests:
memory: '0'
cpu: '0'
jobs:
image_repo_sync:
requests:
memory: '0'
cpu: '0'
limits:
memory: '1024Mi'
cpu: '2000m'
tests:
requests:
memory: '0'
cpu: '0'
limits:
memory: '1024Mi'
cpu: '2000m'
labels:
fluentd:
node_selector_key: fluentd
node_selector_value: enabled
fluentbit:
node_selector_key: fluentbit
node_selector_value: enabled
prometheus_fluentd_exporter:
node_selector_key: openstack-control-plane
node_selector_value: enabled
job:
node_selector_key: openstack-control-plane
node_selector_value: enabled
conf:
fluentbit:
- service:
header: service
Flush: 5
Daemon: Off
Log_Level: info
Parsers_File: parsers.conf
- kernel_messages:
header: input
Name: tail
Tag: kernel
Path: /var/log/kern.log
DB: /var/log/kern.db
Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- kubelet:
header: input
Name: systemd
Tag: journal.*
Path: ${JOURNAL_PATH}
Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
DB: /var/log/kubelet.db
Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- docker_daemon:
header: input
Name: systemd
Tag: journal.*
Path: ${JOURNAL_PATH}
Systemd_Filter: _SYSTEMD_UNIT=docker.service
DB: /var/log/docker.db
Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- libvirt:
header: input
Name: tail
Tag: libvirt
Path: /var/log/libvirt/libvirtd.log
DB: /var/log/libvirt.db
Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- qemu:
header: input
Name: tail
Tag: qemu
Path: /var/log/libvirt/qemu/*.log
DB: /var/log/qemu.db
Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- libvirt_record_modifier:
header: filter
Name: record_modifier
Match: libvirt
Record: hostname ${HOSTNAME}
- qemu_record_modifier:
header: filter
Name: record_modifier
Match: qemu
Record: hostname ${HOSTNAME}
- kernel_record_modifier:
header: filter
Name: record_modifier
Match: kernel
Record: hostname ${HOSTNAME}
- systemd_modify_fields:
header: filter
Name: modify
Match: journal.**
Rename:
_BOOT_ID: BOOT_ID
_CAP_EFFECTIVE: CAP_EFFECTIVE
_CMDLINE: CMDLINE
_COMM: COMM
_EXE: EXE
_GID: GID
_HOSTNAME: HOSTNAME
_MACHINE_ID: MACHINE_ID
_PID: PID
_SYSTEMD_CGROUP: SYSTEMD_CGROUP
_SYSTEMD_SLICE: SYSTEMD_SLICE
_SYSTEMD_UNIT: SYSTEMD_UNIT
_UID: UID
_TRANSPORT: TRANSPORT
- ceph_cluster_logs:
header: input
Name: tail
Tag: ceph.cluster.*
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.log
DB: /var/log/ceph.db
Parsers: syslog
Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- ceph_audit_logs:
header: input
Name: tail
Tag: ceph.audit.*
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.audit.log
DB: /var/log/ceph.db
Parsers: syslog
Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- ceph_mon_logs:
header: input
Name: tail
Tag: ceph.mon.*
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph-mon**.log
DB: /var/log/ceph.db
Parsers: syslog
Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- ceph_osd_logs:
header: input
Name: tail
Tag: ceph.osd.*
Path: /var/log/ceph/airship-ucp-ceph-osd/ceph-osd**.log
DB: /var/log/ceph.db
Parsers: syslog
Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- containers_tail:
header: input
Name: tail
Tag: kube.*
Path: /var/log/containers/*.log
Parser: docker
DB: /var/log/flb_kube.db
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
Mem_Buf_Limit: 5MB
- drop_fluentd_logs:
header: output
Name: "null"
Match: "**.fluentd**"
- kube_filter:
header: filter
Name: kubernetes
Match: kube.*
Merge_JSON_Log: On
- fluentd_output:
header: output
Name: forward
Match: "*"
Host: ${FLUENTD_HOST}
Port: ${FLUENTD_PORT}
fluentd:
- metrics_agent:
header: source
type: monitor_agent
bind: 0.0.0.0
port: 24220
- fluentbit_forward:
header: source
type: forward
port: "#{ENV['FLUENTD_PORT']}"
bind: 0.0.0.0
- filter_fluentd_logs:
header: match
expression: "fluent.**"
type: "null"
- journal_elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "journal.**"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: journal
buffer:
-
- header: buffer
flush_thread_count: 8
flush_interval: "15"
chunk_limit_size: 8MB
queue_limit_length: 256
retry_max_interval: 30
retry_forever: false
- kernel_elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "kernel"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: kernel
buffer:
-
- header: buffer
flush_thread_count: 8
flush_interval: "15"
chunk_limit_size: 8MB
queue_limit_length: 256
retry_max_interval: 30
retry_forever: false
- ceph_elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "**ceph-**.log"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: ceph
buffer:
-
- header: buffer
flush_thread_count: 8
flush_interval: "15"
chunk_limit_size: 8MB
queue_limit_length: 256
retry_max_interval: 30
retry_forever: false
- oslo_fluentd_elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "**.openstack.*"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: openstack
buffer:
-
- header: buffer
flush_thread_count: 8
flush_interval: "15"
chunk_limit_size: 8MB
queue_limit_length: 256
retry_max_interval: 30
retry_forever: false
type_name: oslo_openstack_fluentd
- elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "**"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
buffer:
-
- header: buffer
flush_thread_count: 8
flush_interval: "15"
chunk_limit_size: 8MB
queue_limit_length: 256
retry_max_interval: 30
retry_forever: false
flush_interval: "10"
parsers:
- syslog:
header: parser
Name: syslog
Format: regex
Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
Time_Key: time
Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
Time_Keep: On
Types: "pid:integer"
dependencies:
- osh-infra-helm-toolkit
...