0a1ba88004
This updates the Elasticsearch and Fluent-logging charts to use the most recent configuration keys in their values overrides, and also introduces support for the ceph-rgw s3 api for use for Elasticsearch snapshot repositories Change-Id: Ia998db9006350a22fcc7dc3052301d7a5b8259f4
387 lines
11 KiB
YAML
387 lines
11 KiB
YAML
---
|
|
schema: armada/Chart/v1
|
|
metadata:
|
|
schema: metadata/Document/v1
|
|
name: fluent-logging-global
|
|
layeringDefinition:
|
|
abstract: true
|
|
layer: global
|
|
labels:
|
|
hosttype: fluent-logging-global
|
|
storagePolicy: cleartext
|
|
substitutions:
|
|
# Chart source
|
|
- src:
|
|
schema: pegleg/SoftwareVersions/v1
|
|
name: software-versions
|
|
path: .charts.osh_infra.fluent_logging
|
|
dest:
|
|
path: .source
|
|
|
|
# Images
|
|
- src:
|
|
schema: pegleg/SoftwareVersions/v1
|
|
name: software-versions
|
|
path: .images.osh_infra.fluent_logging
|
|
dest:
|
|
path: .values.images.tags
|
|
|
|
# Endpoints
|
|
- src:
|
|
schema: pegleg/EndpointCatalogue/v1
|
|
name: osh_infra_endpoints
|
|
path: .osh_infra.elasticsearch
|
|
dest:
|
|
path: .values.endpoints.elasticsearch
|
|
- src:
|
|
schema: pegleg/EndpointCatalogue/v1
|
|
name: osh_infra_endpoints
|
|
path: .osh_infra.fluentd
|
|
dest:
|
|
path: .values.endpoints.fluentd
|
|
- src:
|
|
schema: pegleg/EndpointCatalogue/v1
|
|
name: osh_infra_endpoints
|
|
path: .osh_infra.prometheus_fluentd_exporter
|
|
dest:
|
|
path: .values.endpoints.prometheus_fluentd_exporter
|
|
# Accounts
|
|
- src:
|
|
schema: pegleg/AccountCatalogue/v1
|
|
name: osh_infra_service_accounts
|
|
path: .osh_infra.elasticsearch.admin
|
|
dest:
|
|
path: .values.endpoints.elasticsearch.auth.admin
|
|
|
|
# Secrets
|
|
- dest:
|
|
path: .values.endpoints.elasticsearch.auth.admin.password
|
|
src:
|
|
schema: deckhand/Passphrase/v1
|
|
name: osh_infra_elasticsearch_admin_password
|
|
path: .
|
|
|
|
data:
|
|
chart_name: fluent-logging
|
|
release: fluent-logging
|
|
namespace: osh-infra
|
|
wait:
|
|
timeout: 900
|
|
labels:
|
|
release_group: airship-fluent-logging
|
|
install:
|
|
no_hooks: false
|
|
upgrade:
|
|
no_hooks: false
|
|
pre:
|
|
delete:
|
|
- type: job
|
|
labels:
|
|
release_group: airship-fluent-logging
|
|
create: []
|
|
post:
|
|
create: []
|
|
values:
|
|
monitoring:
|
|
prometheus:
|
|
enabled: true
|
|
pod:
|
|
resources:
|
|
enabled: true
|
|
fluentbit:
|
|
limits:
|
|
memory: '4Gi'
|
|
cpu: '2000m'
|
|
requests:
|
|
memory: '2Gi'
|
|
cpu: '1000m'
|
|
fluentd:
|
|
limits:
|
|
memory: '4Gi'
|
|
cpu: '2000m'
|
|
requests:
|
|
memory: '2Gi'
|
|
cpu: '1000m'
|
|
prometheus_fluentd_exporter:
|
|
limits:
|
|
memory: '1024Mi'
|
|
cpu: '2000m'
|
|
requests:
|
|
memory: '0'
|
|
cpu: '0'
|
|
jobs:
|
|
image_repo_sync:
|
|
requests:
|
|
memory: '0'
|
|
cpu: '0'
|
|
limits:
|
|
memory: '1024Mi'
|
|
cpu: '2000m'
|
|
tests:
|
|
requests:
|
|
memory: '0'
|
|
cpu: '0'
|
|
limits:
|
|
memory: '1024Mi'
|
|
cpu: '2000m'
|
|
labels:
|
|
fluentd:
|
|
node_selector_key: fluentd
|
|
node_selector_value: enabled
|
|
fluentbit:
|
|
node_selector_key: fluentbit
|
|
node_selector_value: enabled
|
|
prometheus_fluentd_exporter:
|
|
node_selector_key: openstack-control-plane
|
|
node_selector_value: enabled
|
|
job:
|
|
node_selector_key: openstack-control-plane
|
|
node_selector_value: enabled
|
|
conf:
|
|
fluentbit:
|
|
- service:
|
|
header: service
|
|
Flush: 5
|
|
Daemon: Off
|
|
Log_Level: info
|
|
Parsers_File: parsers.conf
|
|
- kernel_messages:
|
|
header: input
|
|
Name: tail
|
|
Tag: kernel
|
|
Path: /var/log/kern.log
|
|
DB: /var/log/kern.db
|
|
Mem_Buf_Limit: 5MB
|
|
DB.Sync: Normal
|
|
Buffer_Chunk_Size: 1M
|
|
Buffer_Max_Size: 1M
|
|
- kubelet:
|
|
header: input
|
|
Name: systemd
|
|
Tag: journal.*
|
|
Path: ${JOURNAL_PATH}
|
|
Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
|
|
DB: /var/log/kubelet.db
|
|
Mem_Buf_Limit: 5MB
|
|
DB.Sync: Normal
|
|
Buffer_Chunk_Size: 1M
|
|
Buffer_Max_Size: 1M
|
|
- docker_daemon:
|
|
header: input
|
|
Name: systemd
|
|
Tag: journal.*
|
|
Path: ${JOURNAL_PATH}
|
|
Systemd_Filter: _SYSTEMD_UNIT=docker.service
|
|
DB: /var/log/docker.db
|
|
Mem_Buf_Limit: 5MB
|
|
DB.Sync: Normal
|
|
Buffer_Chunk_Size: 1M
|
|
Buffer_Max_Size: 1M
|
|
- kernel_record_modifier:
|
|
header: filter
|
|
Name: record_modifier
|
|
Match: kernel
|
|
Record: hostname ${HOSTNAME}
|
|
- systemd_modify_fields:
|
|
header: filter
|
|
Name: modify
|
|
Match: journal.**
|
|
Rename:
|
|
_BOOT_ID: BOOT_ID
|
|
_CAP_EFFECTIVE: CAP_EFFECTIVE
|
|
_CMDLINE: CMDLINE
|
|
_COMM: COMM
|
|
_EXE: EXE
|
|
_GID: GID
|
|
_HOSTNAME: HOSTNAME
|
|
_MACHINE_ID: MACHINE_ID
|
|
_PID: PID
|
|
_SYSTEMD_CGROUP: SYSTEMD_CGROUP
|
|
_SYSTEMD_SLICE: SYSTEMD_SLICE
|
|
_SYSTEMD_UNIT: SYSTEMD_UNIT
|
|
_UID: UID
|
|
_TRANSPORT: TRANSPORT
|
|
- ceph_cluster_logs:
|
|
header: input
|
|
Name: tail
|
|
Tag: ceph.cluster.*
|
|
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.log
|
|
DB: /var/log/ceph.db
|
|
Parsers: syslog
|
|
Mem_Buf_Limit: 5MB
|
|
DB.Sync: Normal
|
|
Buffer_Chunk_Size: 1M
|
|
Buffer_Max_Size: 1M
|
|
- ceph_audit_logs:
|
|
header: input
|
|
Name: tail
|
|
Tag: ceph.audit.*
|
|
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.audit.log
|
|
DB: /var/log/ceph.db
|
|
Parsers: syslog
|
|
Mem_Buf_Limit: 5MB
|
|
DB.Sync: Normal
|
|
Buffer_Chunk_Size: 1M
|
|
Buffer_Max_Size: 1M
|
|
- ceph_mon_logs:
|
|
header: input
|
|
Name: tail
|
|
Tag: ceph.mon.*
|
|
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph-mon**.log
|
|
DB: /var/log/ceph.db
|
|
Parsers: syslog
|
|
Mem_Buf_Limit: 5MB
|
|
DB.Sync: Normal
|
|
Buffer_Chunk_Size: 1M
|
|
Buffer_Max_Size: 1M
|
|
- ceph_osd_logs:
|
|
header: input
|
|
Name: tail
|
|
Tag: ceph.osd.*
|
|
Path: /var/log/ceph/airship-ucp-ceph-osd/ceph-osd**.log
|
|
DB: /var/log/ceph.db
|
|
Parsers: syslog
|
|
Mem_Buf_Limit: 5MB
|
|
DB.Sync: Normal
|
|
Buffer_Chunk_Size: 1M
|
|
Buffer_Max_Size: 1M
|
|
- containers_tail:
|
|
header: input
|
|
Name: tail
|
|
Tag: kube.*
|
|
Path: /var/log/containers/*.log
|
|
Parser: docker
|
|
DB: /var/log/flb_kube.db
|
|
DB.Sync: Normal
|
|
Buffer_Chunk_Size: 1M
|
|
Buffer_Max_Size: 1M
|
|
Mem_Buf_Limit: 5MB
|
|
- drop_fluentd_logs:
|
|
header: output
|
|
Name: "null"
|
|
Match: "**.fluentd**"
|
|
- kube_filter:
|
|
header: filter
|
|
Name: kubernetes
|
|
Match: kube.*
|
|
Merge_JSON_Log: On
|
|
- fluentd_output:
|
|
header: output
|
|
Name: forward
|
|
Match: "*"
|
|
Host: ${FLUENTD_HOST}
|
|
Port: ${FLUENTD_PORT}
|
|
fluentd:
|
|
- metrics_agent:
|
|
header: source
|
|
type: monitor_agent
|
|
bind: 0.0.0.0
|
|
port: 24220
|
|
- fluentbit_forward:
|
|
header: source
|
|
type: forward
|
|
port: "#{ENV['FLUENTD_PORT']}"
|
|
bind: 0.0.0.0
|
|
- filter_fluentd_logs:
|
|
header: match
|
|
expression: "fluent.**"
|
|
type: "null"
|
|
- journal_elasticsearch:
|
|
header: match
|
|
type: elasticsearch
|
|
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
|
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
|
expression: "journal.**"
|
|
include_tag_key: true
|
|
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
|
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
|
logstash_format: true
|
|
logstash_prefix: journal
|
|
buffer_chunk_limit: 2M
|
|
buffer_queue_limit: 8
|
|
flush_interval: "10"
|
|
max_retry_wait: 300
|
|
disable_retry_limit: ""
|
|
- kernel_elasticsearch:
|
|
header: match
|
|
type: elasticsearch
|
|
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
|
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
|
expression: "kernel"
|
|
include_tag_key: true
|
|
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
|
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
|
logstash_format: true
|
|
logstash_prefix: kernel
|
|
buffer_chunk_limit: 2M
|
|
buffer_queue_limit: 8
|
|
flush_interval: "10"
|
|
max_retry_wait: 300
|
|
disable_retry_limit: ""
|
|
- ceph_elasticsearch:
|
|
header: match
|
|
type: elasticsearch
|
|
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
|
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
|
expression: "**ceph-**.log"
|
|
include_tag_key: true
|
|
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
|
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
|
logstash_format: true
|
|
logstash_prefix: ceph
|
|
buffer_chunk_limit: 10M
|
|
buffer_queue_limit: 32
|
|
flush_interval: "10"
|
|
max_retry_wait: 300
|
|
disable_retry_limit: ""
|
|
num_threads: 8
|
|
- oslo_fluentd_elasticsearch:
|
|
header: match
|
|
type: elasticsearch
|
|
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
|
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
|
expression: "**.openstack.*"
|
|
include_tag_key: true
|
|
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
|
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
|
logstash_format: true
|
|
logstash_prefix: openstack
|
|
buffer_type: memory
|
|
buffer_chunk_limit: 10M
|
|
buffer_queue_limit: 512
|
|
flush_interval: "10"
|
|
max_retry_wait: 300
|
|
request_timeout: 60
|
|
disable_retry_limit: ""
|
|
num_threads: 8
|
|
type_name: oslo_openstack_fluentd
|
|
- elasticsearch:
|
|
header: match
|
|
type: elasticsearch
|
|
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
|
|
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
|
|
expression: "**"
|
|
include_tag_key: true
|
|
host: "#{ENV['ELASTICSEARCH_HOST']}"
|
|
port: "#{ENV['ELASTICSEARCH_PORT']}"
|
|
logstash_format: true
|
|
buffer_chunk_limit: 10M
|
|
buffer_queue_limit: 32g
|
|
flush_interval: "10"
|
|
max_retry_wait: 300
|
|
disable_retry_limit: ""
|
|
num_threads: 8
|
|
parsers:
|
|
- syslog:
|
|
header: parser
|
|
Name: syslog
|
|
Format: regex
|
|
Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
|
|
Time_Key: time
|
|
Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
|
|
Time_Keep: On
|
|
Types: "pid:integer"
|
|
dependencies:
|
|
- osh-infra-helm-toolkit
|
|
...
|