From 4e73195bc2443c202d30869eaa88058185ee5334 Mon Sep 17 00:00:00 2001 From: Steven Fitzpatrick Date: Mon, 27 Jul 2020 08:57:39 -0500 Subject: [PATCH] Fluentd: Allow for Multiple Config Files Change-Id: I2eb6ff2a599ee7bc479f9fc4955016b18902c879 --- fluentd/templates/bin/_fluentd.sh.tpl | 2 +- fluentd/templates/configmap-etc.yaml | 17 +- fluentd/templates/daemonset.yaml | 11 +- fluentd/values.yaml | 282 ++++------------------ tools/deployment/common/fluentd.sh | 335 ++++++++++---------------- 5 files changed, 201 insertions(+), 446 deletions(-) diff --git a/fluentd/templates/bin/_fluentd.sh.tpl b/fluentd/templates/bin/_fluentd.sh.tpl index c689a6ad1..a8caa7aa6 100644 --- a/fluentd/templates/bin/_fluentd.sh.tpl +++ b/fluentd/templates/bin/_fluentd.sh.tpl @@ -18,7 +18,7 @@ set -ex COMMAND="${@:-start}" function start () { - exec fluentd -c /fluentd/etc/fluent.conf + exec fluentd -c /fluentd/etc/main.conf } function stop () { diff --git a/fluentd/templates/configmap-etc.yaml b/fluentd/templates/configmap-etc.yaml index b297394dc..81c112585 100644 --- a/fluentd/templates/configmap-etc.yaml +++ b/fluentd/templates/configmap-etc.yaml @@ -12,8 +12,15 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "fluentd_main" }} +{{- $path := .Values.conf.fluentd.path}} +{{- range $name, $conf := .Values.conf.fluentd.conf }} +{{ printf "%s %s/%s.conf" "@include" $path $name | indent 4}} +{{- end }} +{{- end }} + {{- if .Values.manifests.configmap_etc }} -{{- $envAll := . }} +{{ $envAll := .}} --- apiVersion: v1 kind: Secret @@ -22,6 +29,12 @@ metadata: annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} type: Opaque +stringData: + main.conf: | +{{- template "fluentd_main" . }} data: -{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.fluentd.template "key" "fluent.conf" "format" "Secret") | indent 2 }} +{{- range $name, $config := .Values.conf.fluentd.conf }} +{{- $filename := printf "%s.conf" $name}} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" $config "key" $filename "format" "Secret") | indent 2 }} +{{- end }} {{- end }} diff --git a/fluentd/templates/daemonset.yaml b/fluentd/templates/daemonset.yaml index 4d1037fce..eef101457 100644 --- a/fluentd/templates/daemonset.yaml +++ b/fluentd/templates/daemonset.yaml @@ -15,6 +15,7 @@ limitations under the License. {{- if .Values.manifests.daemonset }} {{- $envAll := . }} +{{- $config_path := .Values.conf.fluentd.path }} {{- $mounts_fluentd := .Values.pod.mounts.fluentd.fluentd }} {{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.fluentd }} @@ -183,9 +184,15 @@ spec: - name: pod-etc-fluentd mountPath: /fluentd/etc - name: fluentd-etc - mountPath: /fluentd/etc/fluent.conf - subPath: fluent.conf + mountPath: {{ printf "%s/%s.conf" $config_path "main" }} + subPath: {{ printf "%s.conf" "main"}} readOnly: true +{{- range $name, $config := .Values.conf.fluentd.conf }} + - name: fluentd-etc + mountPath: {{ printf "%s/%s.conf" $config_path $name }} + subPath: {{ printf "%s.conf" $name }} + readOnly: true +{{- end }} - name: fluentd-bin mountPath: /tmp/fluentd.sh subPath: fluentd.sh diff --git a/fluentd/values.yaml b/fluentd/values.yaml index 90bc833ce..d8b8470a5 100644 --- a/fluentd/values.yaml +++ b/fluentd/values.yaml @@ -54,243 +54,51 @@ dependencies: conf: fluentd: - # This field is now rendered as a helm template! - template: | - - @type prometheus - port {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - - - - @type prometheus_monitor - - - - @type prometheus_output_monitor - - - - @type prometheus_tail_monitor - - - - bind 0.0.0.0 - port "#{ENV['FLUENTD_PORT']}" - @type forward - - - - @type null - - - - - key log - pattern /info/i - tag info.${tag} - - - key log - pattern /warn/i - tag warn.${tag} - - - key log - pattern /error/i - tag error.${tag} - - - key log - pattern /critical/i - tag critical.${tag} - - - key log - pattern (.+) - tag info.${tag} - - @type rewrite_tag_filter - - - - enable_ruby true - - application ${record["kubernetes"]["labels"]["application"]} - level ${tag_parts[0]} - - @type record_transformer - - - - - application ${tag_parts[1]} - - @type record_transformer - - - - - key level - pattern INFO - tag info.${tag} - - - key level - pattern WARN - tag warn.${tag} - - - key level - pattern ERROR - tag error.${tag} - - - key level - pattern CRITICAL - tag critical.${tag} - - @type rewrite_tag_filter - - - - - key application - pattern keystone - tag auth.${tag} - - - key application - pattern horizon - tag auth.${tag} - - - key application - pattern mariadb - tag auth.${tag} - - - key application - pattern memcached - tag auth.${tag} - - - key application - pattern rabbitmq - tag auth.${tag} - - @type rewrite_tag_filter - - - - - chunk_limit_size 8MB - flush_interval 15s - flush_thread_count 8 - queue_limit_length 256 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - reload_connections false - reconnect_on_error true - reload_on_failure true - include_tag_key true - logstash_format true - logstash_prefix libvirt - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 8MB - flush_interval 15s - flush_thread_count 8 - queue_limit_length 256 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - reload_connections false - reconnect_on_error true - reload_on_failure true - include_tag_key true - logstash_format true - logstash_prefix qemu - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 8MB - flush_interval 15s - flush_thread_count 8 - queue_limit_length 256 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - reload_connections false - reconnect_on_error true - reload_on_failure true - include_tag_key true - logstash_format true - logstash_prefix journal - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 8MB - flush_interval 15s - flush_thread_count 8 - queue_limit_length 256 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - reload_connections false - reconnect_on_error true - reload_on_failure true - include_tag_key true - logstash_format true - logstash_prefix kernel - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 8MB - flush_interval 15s - flush_thread_count 8 - queue_limit_length 256 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - reload_connections false - reconnect_on_error true - reload_on_failure true - include_tag_key true - logstash_format true - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - type_name fluent - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - + path: /fluentd/etc + conf: + input: | + + bind 0.0.0.0 + port "#{ENV['FLUENTD_PORT']}" + @type forward + + + + time_format %Y-%m-%dT%H:%M:%S.%NZ + @type json + + path /var/log/containers/*.log + read_from_head true + tag kubernetes.* + @type tail + + + @type relabel + @label @output + + output: | + endpoints: cluster_domain_suffix: cluster.local local_image_registry: diff --git a/tools/deployment/common/fluentd.sh b/tools/deployment/common/fluentd.sh index 5ece3ae86..c7c22b83e 100755 --- a/tools/deployment/common/fluentd.sh +++ b/tools/deployment/common/fluentd.sh @@ -28,228 +28,155 @@ pod: MY_TEST_SECRET: BAR conf: fluentd: - # This field is now rendered as a helm template! - template: | - - @type prometheus - port {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - + conf: + # These fields are rendered as helm templates + input: | + + @type prometheus + port {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - - @type prometheus_monitor - + + @type prometheus_monitor + - - @type prometheus_output_monitor - + + @type prometheus_output_monitor + - - @type prometheus_tail_monitor - + + @type prometheus_tail_monitor + - - bind 0.0.0.0 - port "#{ENV['FLUENTD_PORT']}" - @type forward - + + bind 0.0.0.0 + port "#{ENV['FLUENTD_PORT']}" + @type forward + - - - time_format %Y-%m-%dT%H:%M:%S.%NZ - @type json - - path /var/log/containers/*.log - read_from_head true - tag kubernetes.* - @type tail - + + + time_format %Y-%m-%dT%H:%M:%S.%NZ + @type json + + path /var/log/containers/*.log + read_from_head true + tag kubernetes.* + @type tail + - - @type tail - tag libvirt.* - path /var/log/libvirt/**.log - read_from_head true - - @type none - - + + @type tail + tag libvirt.* + path /var/log/libvirt/**.log + read_from_head true + + @type none + + - - @type systemd - tag auth - path /var/log/journal - matches [{ "SYSLOG_FACILITY":"10" }] - read_from_head true + + @type systemd + tag auth + path /var/log/journal + matches [{ "SYSLOG_FACILITY":"10" }] + read_from_head true - - fields_strip_underscores true - fields_lowercase true - - + + fields_strip_underscores true + fields_lowercase true + + - - @type systemd - tag journal.* - path /var/log/journal - matches [{ "_SYSTEMD_UNIT": "docker.service" }] - read_from_head true + + @type systemd + tag journal.* + path /var/log/journal + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + read_from_head true - - fields_strip_underscores true - fields_lowercase true - - + + fields_strip_underscores true + fields_lowercase true + + - - @type systemd - tag journal.* - path /var/log/journal - matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] - read_from_head true + + @type systemd + tag journal.* + path /var/log/journal + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + read_from_head true - - fields_strip_underscores true - fields_lowercase true - - + + fields_strip_underscores true + fields_lowercase true + + - - @type systemd - tag kernel - path /var/log/journal - matches [{ "_TRANSPORT": "kernel" }] - read_from_head true + + @type systemd + tag kernel + path /var/log/journal + matches [{ "_TRANSPORT": "kernel" }] + read_from_head true - - fields_strip_underscores true - fields_lowercase true - - + + fields_strip_underscores true + fields_lowercase true + + - - @type kubernetes_metadata - + + @type relabel + @label @filter + - - @type record_transformer - - hostname "#{ENV['NODE_NAME']}" - fluentd_pod "#{ENV['POD_NAME']}" - - + filter: | + + output: | + EOF helm upgrade --install fluentd ./fluentd \ --namespace=osh-infra \