Browse Source

Uplift and Separate fluentbit and fluentd charts

fluent-logging chart was splitter into fluentbit and fluentd charts
in the following PS: https://review.opendev.org/642146

Change-Id: Ic72f9fb6293b99395a0acf48064b9c04e312d365
changes/36/667236/16
Alexander Noskov 2 years ago
parent
commit
87b26958a4
  1. 4
      doc/source/airsloop.rst
  2. 13
      global/schemas/pegleg/SoftwareVersions/v1.yaml
  3. 3
      global/software/charts/osh-infra/osh-infra-logging/chart-group.yaml
  4. 419
      global/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml
  5. 255
      global/software/charts/osh-infra/osh-infra-logging/fluentbit.yaml
  6. 375
      global/software/charts/osh-infra/osh-infra-logging/fluentd.yaml
  7. 14
      global/software/config/versions.yaml
  8. 4
      site/airskiff/software/charts/osh-infra/osh-infra-logging/fluentbit.yaml
  9. 18
      site/airskiff/software/charts/osh-infra/osh-infra-logging/fluentd.yaml
  10. 4
      site/seaworthy/software/charts/osh-infra/osh-infra-logging/fluentbit.yaml
  11. 16
      site/seaworthy/software/charts/osh-infra/osh-infra-logging/fluentd.yaml
  12. 6
      type/sloop/charts/osh-infra/fluentbit.yaml
  13. 22
      type/sloop/charts/osh-infra/fluentd.yaml

4
doc/source/airsloop.rst

@ -415,7 +415,9 @@ Under site/airsloop/software/config/ create a versions.yaml file in the followin
osh_infra:
elasticsearch:
proxy_server: proxy.example.com:8080
fluent_logging:
fluentbit:
proxy_server: proxy.example.com:8080
fluentd:
proxy_server: proxy.example.com:8080
grafana:
proxy_server: proxy.example.com:8080

13
global/schemas/pegleg/SoftwareVersions/v1.yaml

@ -235,7 +235,18 @@ data:
type: string
reference:
type: string
fluent_logging:
fluentbit:
type: object
properties:
type:
type: string
location:
type: string
subpath:
type: string
reference:
type: string
fluentd:
type: object
properties:
type:

3
global/software/charts/osh-infra/osh-infra-logging/chart-group.yaml

@ -12,4 +12,5 @@ data:
sequenced: True
chart_group:
- elasticsearch
- fluent-logging
- fluentbit
- fluentd

419
global/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml

@ -1,419 +0,0 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: fluent-logging-global
layeringDefinition:
abstract: true
layer: global
labels:
hosttype: fluent-logging-global
storagePolicy: cleartext
substitutions:
# Chart source
- src:
schema: pegleg/SoftwareVersions/v1
name: software-versions
path: .charts.osh_infra.fluent_logging
dest:
path: .source
# Images
- src:
schema: pegleg/SoftwareVersions/v1
name: software-versions
path: .images.osh_infra.fluent_logging
dest:
path: .values.images.tags
# Endpoints
- src:
schema: pegleg/EndpointCatalogue/v1
name: osh_infra_endpoints
path: .osh_infra.elasticsearch
dest:
path: .values.endpoints.elasticsearch
- src:
schema: pegleg/EndpointCatalogue/v1
name: osh_infra_endpoints
path: .osh_infra.fluentd
dest:
path: .values.endpoints.fluentd
- src:
schema: pegleg/EndpointCatalogue/v1
name: osh_infra_endpoints
path: .osh_infra.prometheus_fluentd_exporter
dest:
path: .values.endpoints.prometheus_fluentd_exporter
# Accounts
- src:
schema: pegleg/AccountCatalogue/v1
name: osh_infra_service_accounts
path: .osh_infra.elasticsearch.admin
dest:
path: .values.endpoints.elasticsearch.auth.admin
# Secrets
- dest:
path: .values.endpoints.elasticsearch.auth.admin.password
src:
schema: deckhand/Passphrase/v1
name: osh_infra_elasticsearch_admin_password
path: .
data:
chart_name: fluent-logging
release: fluent-logging
namespace: osh-infra
wait:
timeout: 900
labels:
release_group: airship-fluent-logging
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete:
- type: job
labels:
release_group: airship-fluent-logging
create: []
post:
create: []
values:
monitoring:
prometheus:
enabled: true
pod:
resources:
enabled: true
fluentbit:
limits:
memory: '4Gi'
cpu: '2000m'
requests:
memory: '2Gi'
cpu: '1000m'
fluentd:
limits:
memory: '4Gi'
cpu: '2000m'
requests:
memory: '2Gi'
cpu: '1000m'
prometheus_fluentd_exporter:
limits:
memory: '1024Mi'
cpu: '2000m'
requests:
memory: '0'
cpu: '0'
jobs:
image_repo_sync:
requests:
memory: '0'
cpu: '0'
limits:
memory: '1024Mi'
cpu: '2000m'
tests:
requests:
memory: '0'
cpu: '0'
limits:
memory: '1024Mi'
cpu: '2000m'
labels:
fluentd:
node_selector_key: fluentd
node_selector_value: enabled
fluentbit:
node_selector_key: fluentbit
node_selector_value: enabled
prometheus_fluentd_exporter:
node_selector_key: openstack-control-plane
node_selector_value: enabled
job:
node_selector_key: openstack-control-plane
node_selector_value: enabled
conf:
fluentbit:
- service:
header: service
Flush: 5
Daemon: Off
Log_Level: info
Parsers_File: parsers.conf
- kernel_messages:
header: input
Name: tail
Tag: kernel
Path: /var/log/kern.log
Mem_Buf_Limit: 5MB
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- kubelet:
header: input
Name: systemd
Tag: journal.*
Path: ${JOURNAL_PATH}
Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
Mem_Buf_Limit: 5MB
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- docker_daemon:
header: input
Name: systemd
Tag: journal.*
Path: ${JOURNAL_PATH}
Systemd_Filter: _SYSTEMD_UNIT=docker.service
Mem_Buf_Limit: 5MB
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- libvirt:
header: input
Name: tail
Tag: libvirt
Path: /var/log/libvirt/libvirtd.log
Mem_Buf_Limit: 5MB
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- qemu:
header: input
Name: tail
Tag: qemu
Path: /var/log/libvirt/qemu/*.log
Mem_Buf_Limit: 5MB
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- libvirt_record_modifier:
header: filter
Name: record_modifier
Match: libvirt
Record: hostname ${HOSTNAME}
- qemu_record_modifier:
header: filter
Name: record_modifier
Match: qemu
Record: hostname ${HOSTNAME}
- kernel_record_modifier:
header: filter
Name: record_modifier
Match: kernel
Record: hostname ${HOSTNAME}
- systemd_modify_fields:
header: filter
Name: modify
Match: journal.**
Rename:
_BOOT_ID: BOOT_ID
_CAP_EFFECTIVE: CAP_EFFECTIVE
_CMDLINE: CMDLINE
_COMM: COMM
_EXE: EXE
_GID: GID
_HOSTNAME: HOSTNAME
_MACHINE_ID: MACHINE_ID
_PID: PID
_SYSTEMD_CGROUP: SYSTEMD_CGROUP
_SYSTEMD_SLICE: SYSTEMD_SLICE
_SYSTEMD_UNIT: SYSTEMD_UNIT
_UID: UID
_TRANSPORT: TRANSPORT
- ceph_cluster_logs:
header: input
Name: tail
Tag: ceph.cluster.*
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.log
Parsers: syslog
Mem_Buf_Limit: 5MB
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- ceph_audit_logs:
header: input
Name: tail
Tag: ceph.audit.*
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.audit.log
Parsers: syslog
Mem_Buf_Limit: 5MB
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- ceph_mon_logs:
header: input
Name: tail
Tag: ceph.mon.*
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph-mon**.log
Parsers: syslog
Mem_Buf_Limit: 5MB
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- ceph_osd_logs:
header: input
Name: tail
Tag: ceph.osd.*
Path: /var/log/ceph/airship-ucp-ceph-osd/ceph-osd**.log
Parsers: syslog
Mem_Buf_Limit: 5MB
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- containers_tail:
header: input
Name: tail
Tag: kube.*
Path: /var/log/containers/*.log
Parser: docker
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
Mem_Buf_Limit: 5MB
- drop_fluentd_logs:
header: output
Name: "null"
Match: "**.fluentd**"
- kube_filter:
header: filter
Name: kubernetes
Match: kube.*
Merge_JSON_Log: On
- throttle_filter:
header: filter
Name: throttle
Match: "**"
Rate: 1000
Window: 300
Interval: 1s
- fluentd_output:
header: output
Name: forward
Match: "*"
Host: ${FLUENTD_HOST}
Port: ${FLUENTD_PORT}
fluentd:
- metrics_agent:
header: source
type: monitor_agent
bind: 0.0.0.0
port: 24220
- fluentbit_forward:
header: source
type: forward
port: "#{ENV['FLUENTD_PORT']}"
bind: 0.0.0.0
- filter_fluentd_logs:
header: match
expression: "fluent.**"
type: "null"
- journal_elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "journal.**"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: journal
buffer:
-
- header: buffer
flush_thread_count: 8
flush_interval: "15"
chunk_limit_size: 8MB
queue_limit_length: 256
retry_max_interval: 30
retry_forever: false
- kernel_elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "kernel"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: kernel
buffer:
-
- header: buffer
flush_thread_count: 8
flush_interval: "15"
chunk_limit_size: 8MB
queue_limit_length: 256
retry_max_interval: 30
retry_forever: false
- ceph_elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "**ceph-**.log"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: ceph
buffer:
-
- header: buffer
flush_thread_count: 8
flush_interval: "15"
chunk_limit_size: 8MB
queue_limit_length: 256
retry_max_interval: 30
retry_forever: false
- oslo_fluentd_elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "**.openstack.*"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: openstack
buffer:
-
- header: buffer
flush_thread_count: 8
flush_interval: "15"
chunk_limit_size: 8MB
queue_limit_length: 256
retry_max_interval: 30
retry_forever: false
type_name: oslo_openstack_fluentd
- elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "**"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
buffer:
-
- header: buffer
flush_thread_count: 8
flush_interval: "15"
chunk_limit_size: 8MB
queue_limit_length: 256
retry_max_interval: 30
retry_forever: false
flush_interval: "10"
parsers:
- syslog:
header: parser
Name: syslog
Format: regex
Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
Time_Key: time
Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
Time_Keep: On
Types: "pid:integer"
dependencies:
- osh-infra-helm-toolkit
...

255
global/software/charts/osh-infra/osh-infra-logging/fluentbit.yaml

@ -0,0 +1,255 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: fluentbit-global
layeringDefinition:
abstract: true
layer: global
labels:
hosttype: fluentbit-global
storagePolicy: cleartext
substitutions:
# Chart source
- src:
schema: pegleg/SoftwareVersions/v1
name: software-versions
path: .charts.osh_infra.fluentbit
dest:
path: .source
# Images
- src:
schema: pegleg/SoftwareVersions/v1
name: software-versions
path: .images.osh_infra.fluentbit
dest:
path: .values.images.tags
# Endpoints
- src:
schema: pegleg/EndpointCatalogue/v1
name: osh_infra_endpoints
path: .osh_infra.fluentd # TODO change it in OSH repo
dest:
path: .values.endpoints.fluentbit
data:
chart_name: fluentbit
release: fluentbit
namespace: osh-infra
wait:
timeout: 900
labels:
release_group: airship-fluentbit
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete:
- type: job
labels:
release_group: airship-fluentbit
create: []
post:
create: []
values:
monitoring:
prometheus:
enabled: true
pod:
resources:
enabled: true
fluentbit:
limits:
memory: '4Gi'
cpu: '2000m'
requests:
memory: '2Gi'
cpu: '1000m'
jobs:
image_repo_sync:
requests:
memory: '0'
cpu: '0'
limits:
memory: '1024Mi'
cpu: '2000m'
tests:
requests:
memory: '0'
cpu: '0'
limits:
memory: '1024Mi'
cpu: '2000m'
labels:
fluentbit:
node_selector_key: fluentbit
node_selector_value: enabled
job:
node_selector_key: openstack-control-plane
node_selector_value: enabled
conf:
fluentbit:
template: |
[SERVICE]
Daemon false
Flush 5
Log_Level info
Parsers_File parsers.conf
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name tail
Path /var/log/kern.log
Tag kernel
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name tail
Parser docker
Path /var/log/containers/*.log
Tag kube.*
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name tail
Path /var/log/libvirt/libvirtd.log
Tag libvirt
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name tail
Path /var/log/libvirt/qemu/*.log
Tag qemu
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name systemd
Path ${JOURNAL_PATH}
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
Tag journal.*
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name systemd
Path ${JOURNAL_PATH}
Systemd_Filter _SYSTEMD_UNIT=docker.service
Tag journal.*
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name tail
Parsers syslog
Path /var/log/ceph/airship-ucp-ceph-mon/ceph.log
Tag ceph.cluster.*
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name tail
Parsers syslog
Path /var/log/ceph/airship-ucp-ceph-mon/ceph.audit.log
Tag ceph.audit.*
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name tail
Parsers syslog
Path /var/log/ceph/airship-ucp-ceph-mon/ceph-mon**.log
Tag ceph.mon.*
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name tail
Parsers syslog
Path /var/log/ceph/airship-ucp-ceph-osd/ceph-osd**.log
Tag ceph.osd.*
[FILTER]
Interval 1s
Match **
Name throttle
Rate 1000
Window 300
[FILTER]
Match libvirt
Name record_modifier
Record hostname ${HOSTNAME}
[FILTER]
Match qemu
Name record_modifier
Record hostname ${HOSTNAME}
[FILTER]
Match kernel
Name record_modifier
Record hostname ${HOSTNAME}
[FILTER]
Match journal.**
Name modify
Rename _BOOT_ID BOOT_ID
Rename _CAP_EFFECTIVE CAP_EFFECTIVE
Rename _CMDLINE CMDLINE
Rename _COMM COMM
Rename _EXE EXE
Rename _GID GID
Rename _HOSTNAME HOSTNAME
Rename _MACHINE_ID MACHINE_ID
Rename _PID PID
Rename _SYSTEMD_CGROUP SYSTEMD_CGROUP
Rename _SYSTEMD_SLICE SYSTEMD_SLICE
Rename _SYSTEMD_UNIT SYSTEMD_UNIT
Rename _TRANSPORT TRANSPORT
Rename _UID UID
[OUTPUT]
Match **.fluentd**
Name null
[FILTER]
Match kube.*
Merge_JSON_Log true
Name kubernetes
[OUTPUT]
Host ${FLUENTD_HOST}
Match *
Name forward
Port ${FLUENTD_PORT}
parsers:
template: |
[PARSER]
Name syslog
Format regex
Regex '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
Time_Key time
Time_Format "%Y-%m-%dT%H:%M:%S.%L"
Time_Keep On
Types "pid:integer"
dependencies:
- osh-infra-helm-toolkit
...

375
global/software/charts/osh-infra/osh-infra-logging/fluentd.yaml

@ -0,0 +1,375 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: fluentd-global
layeringDefinition:
abstract: true
layer: global
labels:
hosttype: fluentd-global
storagePolicy: cleartext
substitutions:
# Chart source
- src:
schema: pegleg/SoftwareVersions/v1
name: software-versions
path: .charts.osh_infra.fluentd
dest:
path: .source
# Images
- src:
schema: pegleg/SoftwareVersions/v1
name: software-versions
path: .images.osh_infra.fluentd
dest:
path: .values.images.tags
# Endpoints
- src:
schema: pegleg/EndpointCatalogue/v1
name: osh_infra_endpoints
path: .osh_infra.fluentd
dest:
path: .values.endpoints.fluentd
- src:
schema: pegleg/EndpointCatalogue/v1
name: osh_infra_endpoints
path: .osh_infra.prometheus_fluentd_exporter
dest:
path: .values.endpoints.prometheus_fluentd_exporter
# Accounts
- src:
schema: pegleg/AccountCatalogue/v1
name: osh_infra_service_accounts
path: .osh_infra.elasticsearch.admin
dest:
path: .values.endpoints.elasticsearch.auth.admin
# Secrets
- dest:
path: .values.endpoints.elasticsearch.auth.admin.password
src:
schema: deckhand/Passphrase/v1
name: osh_infra_elasticsearch_admin_password
path: .
data:
chart_name: fluentd
release: fluentd
namespace: osh-infra
wait:
timeout: 900
labels:
release_group: airship-fluentd
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete:
- type: job
labels:
release_group: airship-fluentd
create: []
post:
create: []
values:
monitoring:
prometheus:
enabled: true
pod:
resources:
enabled: true
fluentd:
limits:
memory: '4Gi'
cpu: '2000m'
requests:
memory: '2Gi'
cpu: '1000m'
prometheus_fluentd_exporter:
limits:
memory: '1024Mi'
cpu: '2000m'
requests:
memory: '0'
cpu: '0'
jobs:
image_repo_sync:
requests:
memory: '0'
cpu: '0'
limits:
memory: '1024Mi'
cpu: '2000m'
tests:
requests:
memory: '0'
cpu: '0'
limits:
memory: '1024Mi'
cpu: '2000m'
labels:
fluentd:
node_selector_key: fluentd
node_selector_value: enabled
prometheus_fluentd_exporter:
node_selector_key: openstack-control-plane
node_selector_value: enabled
job:
node_selector_key: openstack-control-plane
node_selector_value: enabled
conf:
fluentd:
template: |
<source>
bind 0.0.0.0
port 24220
@type monitor_agent
</source>
<source>
bind 0.0.0.0
port "#{ENV['FLUENTD_PORT']}"
@type forward
</source>
<match fluent.**>
@type null
</match>
<match kube.var.log.containers.**.log>
<rule>
key log
pattern /info/i
tag info.${tag}
</rule>
<rule>
key log
pattern /warn/i
tag warn.${tag}
</rule>
<rule>
key log
pattern /error/i
tag error.${tag}
</rule>
<rule>
key log
pattern /critical/i
tag critical.${tag}
</rule>
<rule>
key log
pattern (.+)
tag info.${tag}
</rule>
@type rewrite_tag_filter
</match>
<filter **.kube.var.log.containers.**.log>
enable_ruby true
<record>
application ${record["kubernetes"]["labels"]["application"]}
level ${tag_parts[0]}
</record>
@type record_transformer
</filter>
<filter openstack.**>
<record>
application ${tag_parts[1]}
</record>
@type record_transformer
</filter>
<match openstack.**>
<rule>
key level
pattern INFO
tag info.${tag}
</rule>
<rule>
key level
pattern WARN
tag warn.${tag}
</rule>
<rule>
key level
pattern ERROR
tag error.${tag}
</rule>
<rule>
key level
pattern CRITICAL
tag critical.${tag}
</rule>
@type rewrite_tag_filter
</match>
<match *.openstack.**>
<rule>
key application
pattern keystone
tag auth.${tag}
</rule>
<rule>
key application
pattern horizon
tag auth.${tag}
</rule>
<rule>
key application
pattern mariadb
tag auth.${tag}
</rule>
<rule>
key application
pattern memcached
tag auth.${tag}
</rule>
<rule>
key application
pattern rabbitmq
tag auth.${tag}
</rule>
@type rewrite_tag_filter
</match>
<match libvirt>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix libvirt
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match qemu>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix qemu
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match journal.**>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix journal
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match kernel>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix kernel
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match **>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
type_name fluent
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match *ceph-**.log>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix ceph
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
dependencies:
- osh-infra-helm-toolkit
...

14
global/software/config/versions.yaml

@ -260,10 +260,15 @@ data:
reference: 1f1482bcb71cd3483c6df3ac4bab09653ed6a2e1
subpath: elasticsearch
type: git
fluent_logging:
fluentbit:
location: https://opendev.org/openstack/openstack-helm-infra
reference: 5e1ecd9840397bf9e8829ce0d98fcb721db1b74e
subpath: fluent-logging
reference: 29fc716cf3ff636a66944903fbbe9bd3f921e22f
subpath: fluentbit
type: git
fluentd:
location: https://opendev.org/openstack/openstack-helm-infra
reference: 29fc716cf3ff636a66944903fbbe9bd3f921e22f
subpath: fluentd
type: git
grafana:
location: https://opendev.org/openstack/openstack-helm-infra
@ -592,7 +597,8 @@ data:
tempest: {}
osh_infra:
elasticsearch: {}
fluent_logging: {}
fluentbit: {}
fluentd: {}
grafana: {}
kibana: {}
nagios: {}

4
site/airskiff/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml → site/airskiff/software/charts/osh-infra/osh-infra-logging/fluentbit.yaml

@ -4,12 +4,12 @@
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: fluent-logging
name: fluentbit
layeringDefinition:
abstract: false
layer: site
parentSelector:
hosttype: fluent-logging-global
hosttype: fluentbit-global
actions:
- method: merge
path: .

18
site/airskiff/software/charts/osh-infra/osh-infra-logging/fluentd.yaml

@ -0,0 +1,18 @@
---
# NOTE: This file is ignored by Airskiff and is copied from the seaworthy site.
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: fluentd
layeringDefinition:
abstract: false
layer: site
parentSelector:
hosttype: fluentd-global
actions:
- method: merge
path: .
storagePolicy: cleartext
data: {}
...

4
site/seaworthy/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml → site/seaworthy/software/charts/osh-infra/osh-infra-logging/fluentbit.yaml

@ -2,12 +2,12 @@
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: fluent-logging
name: fluentbit
layeringDefinition:
abstract: false
layer: site
parentSelector:
hosttype: fluent-logging-global
hosttype: fluentbit-global
actions:
- method: merge
path: .

16
site/seaworthy/software/charts/osh-infra/osh-infra-logging/fluentd.yaml

@ -0,0 +1,16 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: fluentd
layeringDefinition:
abstract: false
layer: site
parentSelector:
hosttype: fluentd-global
actions:
- method: merge
path: .
storagePolicy: cleartext
data: {}
...

6
type/sloop/charts/osh-infra/fluent-logging.yaml → type/sloop/charts/osh-infra/fluentbit.yaml

@ -2,14 +2,14 @@
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: fluent-logging
name: fluentbit
labels:
name: fluent-logging-type
name: fluentbit-type
layeringDefinition:
abstract: false
layer: type
parentSelector:
hosttype: fluent-logging-global
hosttype: fluentbit-global
actions:
- method: merge
path: .

22
type/sloop/charts/osh-infra/fluentd.yaml

@ -0,0 +1,22 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: fluentd
labels:
name: fluentd-type
layeringDefinition:
abstract: false
layer: type
parentSelector:
hosttype: fluentd-global
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
values:
pod:
replicas:
fluentd: 1
...
Loading…
Cancel
Save