From fa09705867c3b4ff8fc56f490b82606286154c87 Mon Sep 17 00:00:00 2001
From: Steve Wilkerson <wilkers.steve@gmail.com>
Date: Fri, 14 Sep 2018 17:02:42 -0600
Subject: [PATCH] Fluentbit: Add kernel, kubelet, and dockerd logs

This adds inputs for kernel logs on the host, as well as dockerd
and kubelet logs via the systemd plugin. This also adds a filter
for adding the hostname to the kernel log events, for renaming the
fields for systemd logs as kibana can not visualize fields that
begin with an underscore, and adds elasticsearch indexes for both
kernel and systemd logs

Change-Id: I026470dd45a971047f1e5bd1cd49bd0889589d12
---
 .../templates/bin/_fluent-bit.sh.tpl          |  6 ++
 fluent-logging/values.yaml                    | 90 +++++++++++++++++++
 .../developer/common/130-fluent-logging.sh    | 22 ++++-
 .../multinode/130-fluent-logging.sh           | 20 ++++-
 4 files changed, 136 insertions(+), 2 deletions(-)

diff --git a/fluent-logging/templates/bin/_fluent-bit.sh.tpl b/fluent-logging/templates/bin/_fluent-bit.sh.tpl
index 7745af8e2..106b6fc28 100644
--- a/fluent-logging/templates/bin/_fluent-bit.sh.tpl
+++ b/fluent-logging/templates/bin/_fluent-bit.sh.tpl
@@ -18,4 +18,10 @@ limitations under the License.
 
 set -ex
 
+if [ -d "/var/log/journal" ]; then
+  export JOURNAL_PATH="/var/log/journal"
+else
+  export JOURNAL_PATH="/run/log/journal"
+fi
+
 exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf
diff --git a/fluent-logging/values.yaml b/fluent-logging/values.yaml
index 2030bf8cc..8f728f442 100644
--- a/fluent-logging/values.yaml
+++ b/fluent-logging/values.yaml
@@ -109,6 +109,62 @@ conf:
         Daemon: Off
         Log_Level: info
         Parsers_File: parsers.conf
+    - kernel_messages:
+        header: input
+        Name: tail
+        Tag: kernel
+        Path: /var/log/kern.log
+        DB: /var/log/kern.db
+        Mem_Buf_Limit: 5MB
+        DB.Sync: Normal
+        Buffer_Chunk_Size: 1M
+        Buffer_Max_Size: 1M
+    - kubelet:
+        header: input
+        Name: systemd
+        Tag: journal.*
+        Path: ${JOURNAL_PATH}
+        Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
+        DB: /var/log/kubelet.db
+        Mem_Buf_Limit: 5MB
+        DB.Sync: Normal
+        Buffer_Chunk_Size: 1M
+        Buffer_Max_Size: 1M
+    - docker_daemon:
+        header: input
+        Name: systemd
+        Tag: journal.*
+        Path: ${JOURNAL_PATH}
+        Systemd_Filter: _SYSTEMD_UNIT=docker.service
+        DB: /var/log/docker.db
+        Mem_Buf_Limit: 5MB
+        DB.Sync: Normal
+        Buffer_Chunk_Size: 1M
+        Buffer_Max_Size: 1M
+    - kernel_record_modifier:
+        header: filter
+        Name: record_modifier
+        Match: kernel
+        Record: hostname ${HOSTNAME}
+    - systemd_modify_fields:
+        header: filter
+        Name: modify
+        Match: journal.**
+        Rename:
+          _BOOT_ID: BOOT_ID
+          _CAP_EFFECTIVE: CAP_EFFECTIVE
+          _CMDLINE: CMDLINE
+          _COMM: COMM
+          _EXE: EXE
+          _GID: GID
+          _HOSTNAME: HOSTNAME
+          _MACHINE_ID: MACHINE_ID
+          _PID: PID
+          _SYSTEMD_CGROUP: SYSTEMD_CGROUP
+          _SYSTEMD_SLICE: SYSTEMD_SLICE
+          _SYSTEMD_UNIT: SYSTEMD_UNIT
+          _UID: UID
+          _TRANSPORT: TRANSPORT
     - containers_tail:
         header: input
         Name: tail
@@ -231,6 +287,40 @@ conf:
               key: level
               pattern: CRITICAL
               tag: critical.${tag}
+    - journal_elasticsearch:
+        header: match
+        type: elasticsearch
+        user: "#{ENV['ELASTICSEARCH_USERNAME']}"
+        password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
+        expression: "journal.**"
+        include_tag_key: true
+        host: "#{ENV['ELASTICSEARCH_HOST']}"
+        port: "#{ENV['ELASTICSEARCH_PORT']}"
+        logstash_format: true
+        logstash_prefix: journal
+        buffer_chunk_limit: 10M
+        buffer_queue_limit: 32
+        flush_interval: 20s
+        max_retry_wait: 300
+        disable_retry_limit: ""
+        num_threads: 8
+    - kernel_elasticsearch:
+        header: match
+        type: elasticsearch
+        user: "#{ENV['ELASTICSEARCH_USERNAME']}"
+        password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
+        expression: "kernel"
+        include_tag_key: true
+        host: "#{ENV['ELASTICSEARCH_HOST']}"
+        port: "#{ENV['ELASTICSEARCH_PORT']}"
+        logstash_format: true
+        logstash_prefix: kernel
+        buffer_chunk_limit: 10M
+        buffer_queue_limit: 32
+        flush_interval: 20s
+        max_retry_wait: 300
+        disable_retry_limit: ""
+        num_threads: 8
     - elasticsearch:
         header: match
         type: elasticsearch
diff --git a/tools/deployment/developer/common/130-fluent-logging.sh b/tools/deployment/developer/common/130-fluent-logging.sh
index aa18d24d3..97a0a6d92 100755
--- a/tools/deployment/developer/common/130-fluent-logging.sh
+++ b/tools/deployment/developer/common/130-fluent-logging.sh
@@ -19,10 +19,30 @@ set -xe
 #NOTE: Lint and package chart
 make fluent-logging
 
-#NOTE: Deploy command
+if [ ! -d "/var/log/journal" ]; then
+tee /tmp/fluent-logging.yaml << EOF
+pod:
+  replicas:
+    fluentd: 1
+  mounts:
+    fluentbit:
+      fluentbit:
+        volumes:
+          - name: runlog
+            hostPath:
+              path: /run/log
+        volumeMounts:
+          - name: runlog
+            mountPath: /run/log
+EOF
+helm upgrade --install fluent-logging ./fluent-logging \
+    --namespace=osh-infra \
+    --values=/tmp/fluent-logging.yaml
+else
 helm upgrade --install fluent-logging ./fluent-logging \
     --namespace=osh-infra \
     --set pod.replicas.fluentd=1
+fi
 
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh osh-infra
diff --git a/tools/deployment/multinode/130-fluent-logging.sh b/tools/deployment/multinode/130-fluent-logging.sh
index f3aac3f33..9d9c98390 100755
--- a/tools/deployment/multinode/130-fluent-logging.sh
+++ b/tools/deployment/multinode/130-fluent-logging.sh
@@ -19,10 +19,28 @@ set -xe
 #NOTE: Lint and package chart
 make fluent-logging
 
-#NOTE: Deploy command
+if [ ! -d "/var/log/journal" ]; then
+tee /tmp/fluent-logging.yaml << EOF
+pod:
+  mounts:
+    fluentbit:
+      fluentbit:
+        volumes:
+          - name: runlog
+            hostPath:
+              path: /run/log
+        volumeMounts:
+          - name: runlog
+            mountPath: /run/log
+EOF
+helm upgrade --install fluent-logging ./fluent-logging \
+    --namespace=osh-infra \
+    --values=/tmp/fluent-logging.yaml
+else
 helm upgrade --install fluent-logging ./fluent-logging \
     --namespace=osh-infra \
     --set monitoring.prometheus.enabled=true
+fi
 
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh osh-infra