Fluentd: Allow for Multiple Config Files

Change-Id: I2eb6ff2a599ee7bc479f9fc4955016b18902c879
This commit is contained in:
Steven Fitzpatrick 2020-07-27 08:57:39 -05:00
parent 959417f321
commit 4e73195bc2
5 changed files with 201 additions and 446 deletions

View File

@ -18,7 +18,7 @@ set -ex
COMMAND="${@:-start}"
function start () {
exec fluentd -c /fluentd/etc/fluent.conf
exec fluentd -c /fluentd/etc/main.conf
}
function stop () {

View File

@ -12,8 +12,15 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "fluentd_main" }}
{{- $path := .Values.conf.fluentd.path}}
{{- range $name, $conf := .Values.conf.fluentd.conf }}
{{ printf "%s %s/%s.conf" "@include" $path $name | indent 4}}
{{- end }}
{{- end }}
{{- if .Values.manifests.configmap_etc }}
{{- $envAll := . }}
{{ $envAll := .}}
---
apiVersion: v1
kind: Secret
@ -22,6 +29,12 @@ metadata:
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
type: Opaque
stringData:
main.conf: |
{{- template "fluentd_main" . }}
data:
{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.fluentd.template "key" "fluent.conf" "format" "Secret") | indent 2 }}
{{- range $name, $config := .Values.conf.fluentd.conf }}
{{- $filename := printf "%s.conf" $name}}
{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" $config "key" $filename "format" "Secret") | indent 2 }}
{{- end }}
{{- end }}

View File

@ -15,6 +15,7 @@ limitations under the License.
{{- if .Values.manifests.daemonset }}
{{- $envAll := . }}
{{- $config_path := .Values.conf.fluentd.path }}
{{- $mounts_fluentd := .Values.pod.mounts.fluentd.fluentd }}
{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.fluentd }}
@ -183,9 +184,15 @@ spec:
- name: pod-etc-fluentd
mountPath: /fluentd/etc
- name: fluentd-etc
mountPath: /fluentd/etc/fluent.conf
subPath: fluent.conf
mountPath: {{ printf "%s/%s.conf" $config_path "main" }}
subPath: {{ printf "%s.conf" "main"}}
readOnly: true
{{- range $name, $config := .Values.conf.fluentd.conf }}
- name: fluentd-etc
mountPath: {{ printf "%s/%s.conf" $config_path $name }}
subPath: {{ printf "%s.conf" $name }}
readOnly: true
{{- end }}
- name: fluentd-bin
mountPath: /tmp/fluentd.sh
subPath: fluentd.sh

View File

@ -54,243 +54,51 @@ dependencies:
conf:
fluentd:
# This field is now rendered as a helm template!
template: |
<source>
@type prometheus
port {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
</source>
<source>
@type prometheus_monitor
</source>
<source>
@type prometheus_output_monitor
</source>
<source>
@type prometheus_tail_monitor
</source>
<source>
bind 0.0.0.0
port "#{ENV['FLUENTD_PORT']}"
@type forward
</source>
<match fluent.**>
@type null
</match>
<match kube.var.log.containers.**.log>
<rule>
key log
pattern /info/i
tag info.${tag}
</rule>
<rule>
key log
pattern /warn/i
tag warn.${tag}
</rule>
<rule>
key log
pattern /error/i
tag error.${tag}
</rule>
<rule>
key log
pattern /critical/i
tag critical.${tag}
</rule>
<rule>
key log
pattern (.+)
tag info.${tag}
</rule>
@type rewrite_tag_filter
</match>
<filter **.kube.var.log.containers.**.log>
enable_ruby true
<record>
application ${record["kubernetes"]["labels"]["application"]}
level ${tag_parts[0]}
</record>
@type record_transformer
</filter>
<filter openstack.**>
<record>
application ${tag_parts[1]}
</record>
@type record_transformer
</filter>
<match openstack.**>
<rule>
key level
pattern INFO
tag info.${tag}
</rule>
<rule>
key level
pattern WARN
tag warn.${tag}
</rule>
<rule>
key level
pattern ERROR
tag error.${tag}
</rule>
<rule>
key level
pattern CRITICAL
tag critical.${tag}
</rule>
@type rewrite_tag_filter
</match>
<match *.openstack.**>
<rule>
key application
pattern keystone
tag auth.${tag}
</rule>
<rule>
key application
pattern horizon
tag auth.${tag}
</rule>
<rule>
key application
pattern mariadb
tag auth.${tag}
</rule>
<rule>
key application
pattern memcached
tag auth.${tag}
</rule>
<rule>
key application
pattern rabbitmq
tag auth.${tag}
</rule>
@type rewrite_tag_filter
</match>
<match libvirt>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix libvirt
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match qemu>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix qemu
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match journal.**>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix journal
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match kernel>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix kernel
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match **>
<buffer>
chunk_limit_size 8MB
flush_interval 15s
flush_thread_count 8
queue_limit_length 256
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
type_name fluent
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
path: /fluentd/etc
conf:
input: |
<source>
bind 0.0.0.0
port "#{ENV['FLUENTD_PORT']}"
@type forward
</source>
<source>
<parse>
time_format %Y-%m-%dT%H:%M:%S.%NZ
@type json
</parse>
path /var/log/containers/*.log
read_from_head true
tag kubernetes.*
@type tail
</source>
<match **>
@type relabel
@label @output
</match>
output: |
<label @output>
<match **>
<buffer>
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
</label>
endpoints:
cluster_domain_suffix: cluster.local
local_image_registry:

View File

@ -28,228 +28,155 @@ pod:
MY_TEST_SECRET: BAR
conf:
fluentd:
# This field is now rendered as a helm template!
template: |
<source>
@type prometheus
port {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
</source>
conf:
# These fields are rendered as helm templates
input: |
<source>
@type prometheus
port {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
</source>
<source>
@type prometheus_monitor
</source>
<source>
@type prometheus_monitor
</source>
<source>
@type prometheus_output_monitor
</source>
<source>
@type prometheus_output_monitor
</source>
<source>
@type prometheus_tail_monitor
</source>
<source>
@type prometheus_tail_monitor
</source>
<source>
bind 0.0.0.0
port "#{ENV['FLUENTD_PORT']}"
@type forward
</source>
<source>
bind 0.0.0.0
port "#{ENV['FLUENTD_PORT']}"
@type forward
</source>
<source>
<parse>
time_format %Y-%m-%dT%H:%M:%S.%NZ
@type json
</parse>
path /var/log/containers/*.log
read_from_head true
tag kubernetes.*
@type tail
</source>
<source>
<parse>
time_format %Y-%m-%dT%H:%M:%S.%NZ
@type json
</parse>
path /var/log/containers/*.log
read_from_head true
tag kubernetes.*
@type tail
</source>
<source>
@type tail
tag libvirt.*
path /var/log/libvirt/**.log
read_from_head true
<parse>
@type none
</parse>
</source>
<source>
@type tail
tag libvirt.*
path /var/log/libvirt/**.log
read_from_head true
<parse>
@type none
</parse>
</source>
<source>
@type systemd
tag auth
path /var/log/journal
matches [{ "SYSLOG_FACILITY":"10" }]
read_from_head true
<source>
@type systemd
tag auth
path /var/log/journal
matches [{ "SYSLOG_FACILITY":"10" }]
read_from_head true
<entry>
fields_strip_underscores true
fields_lowercase true
</entry>
</source>
<entry>
fields_strip_underscores true
fields_lowercase true
</entry>
</source>
<source>
@type systemd
tag journal.*
path /var/log/journal
matches [{ "_SYSTEMD_UNIT": "docker.service" }]
read_from_head true
<source>
@type systemd
tag journal.*
path /var/log/journal
matches [{ "_SYSTEMD_UNIT": "docker.service" }]
read_from_head true
<entry>
fields_strip_underscores true
fields_lowercase true
</entry>
</source>
<entry>
fields_strip_underscores true
fields_lowercase true
</entry>
</source>
<source>
@type systemd
tag journal.*
path /var/log/journal
matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
read_from_head true
<source>
@type systemd
tag journal.*
path /var/log/journal
matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
read_from_head true
<entry>
fields_strip_underscores true
fields_lowercase true
</entry>
</source>
<entry>
fields_strip_underscores true
fields_lowercase true
</entry>
</source>
<source>
@type systemd
tag kernel
path /var/log/journal
matches [{ "_TRANSPORT": "kernel" }]
read_from_head true
<source>
@type systemd
tag kernel
path /var/log/journal
matches [{ "_TRANSPORT": "kernel" }]
read_from_head true
<entry>
fields_strip_underscores true
fields_lowercase true
</entry>
</source>
<entry>
fields_strip_underscores true
fields_lowercase true
</entry>
</source>
<filter kubernetes.**>
@type kubernetes_metadata
</filter>
<match **>
@type relabel
@label @filter
</match>
<filter libvirt.**>
@type record_transformer
<record>
hostname "#{ENV['NODE_NAME']}"
fluentd_pod "#{ENV['POD_NAME']}"
</record>
</filter>
filter: |
<label @filter>
<filter kubernetes.**>
@type kubernetes_metadata
</filter>
<match fluent.**>
@type null
</match>
<filter libvirt.**>
@type record_transformer
<record>
hostname "#{ENV['NODE_NAME']}"
fluentd_pod "#{ENV['POD_NAME']}"
</record>
</filter>
<match **>
@type relabel
@label @output
</match>
</label>
output: |
<label @output>
<match fluent.**>
@type null
</match>
<match libvirt.**>
<buffer>
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix libvirt
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match kernel>
<buffer>
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 32
retry_forever false
disable_chunk_backup true
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix kernel
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match auth>
<buffer>
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix auth
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match journal.**>
<buffer>
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
logstash_prefix journal
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match **>
<buffer>
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
<match **>
<buffer>
chunk_limit_size 512K
flush_interval 5s
flush_thread_count 8
queue_limit_length 32
retry_forever false
retry_max_interval 30
</buffer>
host "#{ENV['ELASTICSEARCH_HOST']}"
reload_connections false
reconnect_on_error true
reload_on_failure true
include_tag_key true
logstash_format true
password "#{ENV['ELASTICSEARCH_PASSWORD']}"
port "#{ENV['ELASTICSEARCH_PORT']}"
@type elasticsearch
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
</label>
EOF
helm upgrade --install fluentd ./fluentd \
--namespace=osh-infra \