From fa09705867c3b4ff8fc56f490b82606286154c87 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson <wilkers.steve@gmail.com> Date: Fri, 14 Sep 2018 17:02:42 -0600 Subject: [PATCH] Fluentbit: Add kernel, kubelet, and dockerd logs This adds inputs for kernel logs on the host, as well as dockerd and kubelet logs via the systemd plugin. This also adds a filter for adding the hostname to the kernel log events, for renaming the fields for systemd logs as kibana can not visualize fields that begin with an underscore, and adds elasticsearch indexes for both kernel and systemd logs Change-Id: I026470dd45a971047f1e5bd1cd49bd0889589d12 --- .../templates/bin/_fluent-bit.sh.tpl | 6 ++ fluent-logging/values.yaml | 90 +++++++++++++++++++ .../developer/common/130-fluent-logging.sh | 22 ++++- .../multinode/130-fluent-logging.sh | 20 ++++- 4 files changed, 136 insertions(+), 2 deletions(-) diff --git a/fluent-logging/templates/bin/_fluent-bit.sh.tpl b/fluent-logging/templates/bin/_fluent-bit.sh.tpl index 7745af8e2..106b6fc28 100644 --- a/fluent-logging/templates/bin/_fluent-bit.sh.tpl +++ b/fluent-logging/templates/bin/_fluent-bit.sh.tpl @@ -18,4 +18,10 @@ limitations under the License. set -ex +if [ -d "/var/log/journal" ]; then + export JOURNAL_PATH="/var/log/journal" +else + export JOURNAL_PATH="/run/log/journal" +fi + exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf diff --git a/fluent-logging/values.yaml b/fluent-logging/values.yaml index 2030bf8cc..8f728f442 100644 --- a/fluent-logging/values.yaml +++ b/fluent-logging/values.yaml @@ -109,6 +109,62 @@ conf: Daemon: Off Log_Level: info Parsers_File: parsers.conf + - kernel_messages: + header: input + Name: tail + Tag: kernel + Path: /var/log/kern.log + DB: /var/log/kern.db + Mem_Buf_Limit: 5MB + DB.Sync: Normal + Buffer_Chunk_Size: 1M + Buffer_Max_Size: 1M + - kubelet: + header: input + Name: systemd + Tag: journal.* + Path: ${JOURNAL_PATH} + Systemd_Filter: _SYSTEMD_UNIT=kubelet.service + DB: /var/log/kubelet.db + Mem_Buf_Limit: 5MB + DB.Sync: Normal + Buffer_Chunk_Size: 1M + Buffer_Max_Size: 1M + - docker_daemon: + header: input + Name: systemd + Tag: journal.* + Path: ${JOURNAL_PATH} + Systemd_Filter: _SYSTEMD_UNIT=docker.service + DB: /var/log/docker.db + Mem_Buf_Limit: 5MB + DB.Sync: Normal + Buffer_Chunk_Size: 1M + Buffer_Max_Size: 1M + - kernel_record_modifier: + header: filter + Name: record_modifier + Match: kernel + Record: hostname ${HOSTNAME} + - systemd_modify_fields: + header: filter + Name: modify + Match: journal.** + Rename: + _BOOT_ID: BOOT_ID + _CAP_EFFECTIVE: CAP_EFFECTIVE + _CMDLINE: CMDLINE + _COMM: COMM + _EXE: EXE + _GID: GID + _HOSTNAME: HOSTNAME + _MACHINE_ID: MACHINE_ID + _PID: PID + _SYSTEMD_CGROUP: SYSTEMD_CGROUP + _SYSTEMD_SLICE: SYSTEMD_SLICE + _SYSTEMD_UNIT: SYSTEMD_UNIT + _UID: UID + _TRANSPORT: TRANSPORT - containers_tail: header: input Name: tail @@ -231,6 +287,40 @@ conf: key: level pattern: CRITICAL tag: critical.${tag} + - journal_elasticsearch: + header: match + type: elasticsearch + user: "#{ENV['ELASTICSEARCH_USERNAME']}" + password: "#{ENV['ELASTICSEARCH_PASSWORD']}" + expression: "journal.**" + include_tag_key: true + host: "#{ENV['ELASTICSEARCH_HOST']}" + port: "#{ENV['ELASTICSEARCH_PORT']}" + logstash_format: true + logstash_prefix: journal + buffer_chunk_limit: 10M + buffer_queue_limit: 32 + flush_interval: 20s + max_retry_wait: 300 + disable_retry_limit: "" + num_threads: 8 + - kernel_elasticsearch: + header: match + type: elasticsearch + user: "#{ENV['ELASTICSEARCH_USERNAME']}" + password: "#{ENV['ELASTICSEARCH_PASSWORD']}" + expression: "kernel" + include_tag_key: true + host: "#{ENV['ELASTICSEARCH_HOST']}" + port: "#{ENV['ELASTICSEARCH_PORT']}" + logstash_format: true + logstash_prefix: kernel + buffer_chunk_limit: 10M + buffer_queue_limit: 32 + flush_interval: 20s + max_retry_wait: 300 + disable_retry_limit: "" + num_threads: 8 - elasticsearch: header: match type: elasticsearch diff --git a/tools/deployment/developer/common/130-fluent-logging.sh b/tools/deployment/developer/common/130-fluent-logging.sh index aa18d24d3..97a0a6d92 100755 --- a/tools/deployment/developer/common/130-fluent-logging.sh +++ b/tools/deployment/developer/common/130-fluent-logging.sh @@ -19,10 +19,30 @@ set -xe #NOTE: Lint and package chart make fluent-logging -#NOTE: Deploy command +if [ ! -d "/var/log/journal" ]; then +tee /tmp/fluent-logging.yaml << EOF +pod: + replicas: + fluentd: 1 + mounts: + fluentbit: + fluentbit: + volumes: + - name: runlog + hostPath: + path: /run/log + volumeMounts: + - name: runlog + mountPath: /run/log +EOF +helm upgrade --install fluent-logging ./fluent-logging \ + --namespace=osh-infra \ + --values=/tmp/fluent-logging.yaml +else helm upgrade --install fluent-logging ./fluent-logging \ --namespace=osh-infra \ --set pod.replicas.fluentd=1 +fi #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh osh-infra diff --git a/tools/deployment/multinode/130-fluent-logging.sh b/tools/deployment/multinode/130-fluent-logging.sh index f3aac3f33..9d9c98390 100755 --- a/tools/deployment/multinode/130-fluent-logging.sh +++ b/tools/deployment/multinode/130-fluent-logging.sh @@ -19,10 +19,28 @@ set -xe #NOTE: Lint and package chart make fluent-logging -#NOTE: Deploy command +if [ ! -d "/var/log/journal" ]; then +tee /tmp/fluent-logging.yaml << EOF +pod: + mounts: + fluentbit: + fluentbit: + volumes: + - name: runlog + hostPath: + path: /run/log + volumeMounts: + - name: runlog + mountPath: /run/log +EOF +helm upgrade --install fluent-logging ./fluent-logging \ + --namespace=osh-infra \ + --values=/tmp/fluent-logging.yaml +else helm upgrade --install fluent-logging ./fluent-logging \ --namespace=osh-infra \ --set monitoring.prometheus.enabled=true +fi #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh osh-infra