Merge "Fluentd: Support Daemonset deployment"

doc/source/install/multinode.rst

@@ -226,7 +226,7 @@ Alternatively, this step can be performed by running the script directly:
 Deploy Fluentd
 ^^^^^^^^^^^^^^
 
-.. literalinclude:: ../../../tools/deployment/multinode/130-fluentd.sh
+.. literalinclude:: ../../../tools/deployment/multinode/135-fluentd-deployment.sh
     :language: shell
     :lines: 1,17-
 
@@ -234,4 +234,4 @@ Alternatively, this step can be performed by running the script directly:
 
 .. code-block:: shell
 
-  ./tools/deployment/multinode/130-fluentd.sh
+  ./tools/deployment/multinode/135-fluentd-deployment.sh

fluentd/templates/configmap-bin.yaml

@@ -20,7 +20,7 @@ limitations under the License.
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: fluentd-bin
+  name: {{ printf "%s-%s" $envAll.Release.Name "fluentd-bin" | quote }}
 data:
   fluentd.sh: |
 {{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

fluentd/templates/configmap-etc.yaml

@@ -20,7 +20,7 @@ limitations under the License.
 apiVersion: v1
 kind: Secret
 metadata:
-  name: fluentd-etc
+  name: {{ printf "%s-%s" $envAll.Release.Name "fluentd-etc" | quote }}
 type: Opaque
 data:
   fluent.conf: {{ .Values.conf.fluentd.template | b64enc }}

fluentd/templates/deployment-fluentd.yaml

@@ -16,30 +16,31 @@ limitations under the License.
 
 {{- if .Values.manifests.deployment_fluentd }}
 {{- $envAll := . }}
-{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 
 {{- $mounts_fluentd := .Values.pod.mounts.fluentd.fluentd }}
 
-{{- $serviceAccountName := printf "%s-%s" .Release.Name "fluentd" }}
-{{ tuple $envAll "fluentd" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+{{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "fluentd"  }}
+
+{{ tuple $envAll "fluentd" $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
 metadata:
-  name: {{ $serviceAccountName }}
+  name: {{ $rcControllerName | quote }}
 subjects:
   - kind: ServiceAccount
-    name: {{ $serviceAccountName }}
+    name: {{ $rcControllerName | quote }}
     namespace: {{ .Release.Namespace }}
 roleRef:
   kind: ClusterRole
-  name: {{ $serviceAccountName }}
+  name: {{ $rcControllerName | quote }}
   apiGroup: rbac.authorization.k8s.io
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: {{ $serviceAccountName }}
+  name: {{ $rcControllerName | quote }}
+rules:
 rules:
   - apiGroups:
       - ""
@@ -51,7 +52,6 @@ rules:
       - replicationcontrollers
       - limitranges
     verbs:
-      - get
       - list
       - watch
   - apiGroups:
@@ -72,20 +72,27 @@ rules:
       - list
       - watch
 ---
+{{- if eq .Values.deployment.type "Deployment" }}
 apiVersion: apps/v1
 kind: Deployment
+{{- else if eq .Values.deployment.type "DaemonSet" }}
+apiVersion: apps/v1
+kind: DaemonSet
+{{- end }}
 metadata:
-  name: fluentd
+  name: {{ $rcControllerName | quote }}
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
   labels:
 {{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 spec:
+{{- if eq .Values.deployment.type "Deployment" }}
   replicas: {{ .Values.pod.replicas.fluentd }}
+{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
+{{- end }}
   selector:
     matchLabels:
 {{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
-{{ tuple $envAll  | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
   template:
     metadata:
       labels:
@@ -96,9 +103,14 @@ spec:
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
     spec:
 {{ dict "envAll" $envAll "application" "fluentd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
-      serviceAccountName: {{ $serviceAccountName }}
+      serviceAccountName: {{ $rcControllerName | quote }}
+{{ if and ($envAll.Values.pod.tolerations.fluentd.enabled) (eq .Values.deployment.type "DaemonSet") }}
+{{ tuple $envAll "fluentd" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
+{{ end }}
+{{- if eq .Values.deployment.type "Deployment" }}
       affinity:
 {{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
+{{- end }}
       nodeSelector:
         {{ .Values.labels.fluentd.node_selector_key }}: {{ .Values.labels.fluentd.node_selector_value | quote }}
       terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.fluentd.timeout | default "30" }}
@@ -137,16 +149,24 @@ spec:
             - name: ELASTICSEARCH_USERNAME
               valueFrom:
                 secretKeyRef:
-                  name: {{ $esUserSecret }}
+                  name: {{ printf "%s-%s" $envAll.Release.Name "elasticsearch-user" | quote }}
                   key: ELASTICSEARCH_USERNAME
             - name: ELASTICSEARCH_PASSWORD
               valueFrom:
                 secretKeyRef:
-                  name: {{ $esUserSecret }}
+                  name: {{ printf "%s-%s" $envAll.Release.Name "elasticsearch-user" | quote }}
                   key: ELASTICSEARCH_PASSWORD
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
+{{- if eq .Values.deployment.type "DaemonSet" }}
+            - name: varlog
+              mountPath: /var/log
+              readOnly: true
+            - name: varlibdockercontainers
+              mountPath: /var/lib/docker/containers
+              readOnly: true
+{{- end }}
             - name: pod-etc-fluentd
               mountPath: /fluentd/etc
             - name: fluentd-etc
@@ -161,15 +181,23 @@ spec:
       volumes:
         - name: pod-tmp
           emptyDir: {}
+{{- if eq .Values.deployment.type "DaemonSet" }}
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers
+{{- end }}
         - name: pod-etc-fluentd
           emptyDir: {}
         - name: fluentd-etc
           secret:
-            secretName: fluentd-etc
+            secretName: {{ printf "%s-%s" $envAll.Release.Name "fluentd-etc" | quote }}
             defaultMode: 0444
         - name: fluentd-bin
           configMap:
-            name: fluentd-bin
+            name: {{ printf "%s-%s" $envAll.Release.Name "fluentd-bin" | quote }}
             defaultMode: 0555
 {{- if $mounts_fluentd.volumes }}{{ toYaml $mounts_fluentd.volumes | indent 8 }}{{- end }}
 {{- end }}

fluentd/templates/monitoring/prometheus/exporter-configmap-bin.yaml

@@ -20,7 +20,7 @@ limitations under the License.
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: fluentd-exporter-bin
+  name: {{ printf "%s-%s" $envAll.Release.Name "fluentd-exporter-bin" }}
 data:
   fluentd-exporter.sh: |
 {{ tuple "bin/_fluentd-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

fluentd/templates/monitoring/prometheus/exporter-deployment.yaml

@@ -21,13 +21,14 @@ limitations under the License.
 {{ $fluentd_metrics_path := "api/plugins.json" }}
 {{ $fluentd_metrics_host := printf "http://%s/%s" $fluentd_host $fluentd_metrics_path }}
 
-{{- $serviceAccountName := "prometheus-fluentd-exporter" }}
-{{ tuple $envAll "prometheus_fluentd_exporter" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+{{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "fluentd-exporter"  }}
+
+{{ tuple $envAll "prometheus_fluentd_exporter" $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: prometheus-fluentd-exporter
+  name: {{ $rcControllerName | quote }}
   labels:
 {{ tuple $envAll "prometheus-fluentd-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 spec:
@@ -42,7 +43,7 @@ spec:
 {{ tuple $envAll "prometheus-fluentd-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
-      serviceAccountName: {{ $serviceAccountName }}
+      serviceAccountName: {{ $rcControllerName | quote }}
       nodeSelector:
         {{ .Values.labels.prometheus_fluentd_exporter.node_selector_key }}: {{ .Values.labels.prometheus_fluentd_exporter.node_selector_value | quote }}
       terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.prometheus_fluentd_exporter.timeout | default "30" }}
@@ -79,6 +80,6 @@ spec:
           emptyDir: {}
         - name: fluentd-exporter-bin
           configMap:
-            name: fluentd-exporter-bin
+            name: {{ printf "%s-%s" $envAll.Release.Name "fluentd-exporter-bin" | quote }}
             defaultMode: 0555
 {{- end }}

fluentd/templates/secret-elasticsearch-creds.yaml

@@ -16,12 +16,11 @@ limitations under the License.
 
 {{- if .Values.manifests.secret_elasticsearch }}
 {{- $envAll := . }}
-{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }}
 ---
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ $secretName }}
+  name: {{ printf "%s-%s" $envAll.Release.Name "elasticsearch-user" | quote }}
 type: Opaque
 data:
   ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }}

fluentd/values.yaml

@@ -18,6 +18,9 @@
 
 release_group: null
 
+deployment:
+  type: DaemonSet
+
 labels:
   fluentd:
     node_selector_key: openstack-control-plane
@@ -41,10 +44,6 @@ images:
       - dep_check
       - image_repo_sync
 
-secrets:
-  elasticsearch:
-    user: fluentd-elasticsearch-user
-
 dependencies:
   dynamic:
     common:
@@ -278,128 +277,6 @@ conf:
         type_name fluent
         user "#{ENV['ELASTICSEARCH_USERNAME']}"
       </match>
-  fluentbit:
-    template: |
-      [SERVICE]
-          Daemon false
-          Flush 30
-          Log_Level info
-          Parsers_File parsers.conf
-
-      [INPUT]
-          Buffer_Chunk_Size 1M
-          Buffer_Max_Size 1M
-          Mem_Buf_Limit 5MB
-          Name tail
-          Path /var/log/kern.log
-          Tag kernel
-
-      [INPUT]
-          Buffer_Chunk_Size 1M
-          Buffer_Max_Size 1M
-          Mem_Buf_Limit 5MB
-          Name tail
-          Parser docker
-          Path /var/log/containers/*.log
-          Tag kube.*
-
-      [INPUT]
-          Buffer_Chunk_Size 1M
-          Buffer_Max_Size 1M
-          Mem_Buf_Limit 5MB
-          Name tail
-          Path /var/log/libvirt/libvirtd.log
-          Tag libvirt
-
-      [INPUT]
-          Buffer_Chunk_Size 1M
-          Buffer_Max_Size 1M
-          Mem_Buf_Limit 5MB
-          Name tail
-          Path /var/log/libvirt/qemu/*.log
-          Tag qemu
-
-      [INPUT]
-          Buffer_Chunk_Size 1M
-          Buffer_Max_Size 1M
-          Mem_Buf_Limit 5MB
-          Name systemd
-          Path ${JOURNAL_PATH}
-          Systemd_Filter _SYSTEMD_UNIT=kubelet.service
-          Tag journal.*
-
-      [INPUT]
-          Buffer_Chunk_Size 1M
-          Buffer_Max_Size 1M
-          Mem_Buf_Limit 5MB
-          Name systemd
-          Path ${JOURNAL_PATH}
-          Systemd_Filter _SYSTEMD_UNIT=docker.service
-          Tag journal.*
-
-      [FILTER]
-          Interval 1s
-          Match **
-          Name throttle
-          Rate 1000
-          Window 300
-
-      [FILTER]
-          Match libvirt
-          Name record_modifier
-          Record hostname ${HOSTNAME}
-
-      [FILTER]
-          Match qemu
-          Name record_modifier
-          Record hostname ${HOSTNAME}
-
-      [FILTER]
-          Match kernel
-          Name record_modifier
-          Record hostname ${HOSTNAME}
-
-      [FILTER]
-          Match journal.**
-          Name modify
-          Rename _BOOT_ID BOOT_ID
-          Rename _CAP_EFFECTIVE CAP_EFFECTIVE
-          Rename _CMDLINE CMDLINE
-          Rename _COMM COMM
-          Rename _EXE EXE
-          Rename _GID GID
-          Rename _HOSTNAME HOSTNAME
-          Rename _MACHINE_ID MACHINE_ID
-          Rename _PID PID
-          Rename _SYSTEMD_CGROUP SYSTEMD_CGROUP
-          Rename _SYSTEMD_SLICE SYSTEMD_SLICE
-          Rename _SYSTEMD_UNIT SYSTEMD_UNIT
-          Rename _TRANSPORT TRANSPORT
-          Rename _UID UID
-
-      [OUTPUT]
-          Match **.fluentd**
-          Name null
-
-      [FILTER]
-          Match kube.*
-          Merge_JSON_Log true
-          Name kubernetes
-
-      [OUTPUT]
-          Host ${FLUENTD_HOST}
-          Match *
-          Name forward
-          Port ${FLUENTD_PORT}
-  parsers:
-    template: |
-      [PARSER]
-        Decode_Field_As escaped_utf8 log
-        Format json
-        Name docker
-        Time_Format %Y-%m-%dT%H:%M:%S.%L
-        Time_Keep true
-        Time_Key time
   fluentd_exporter:
     log:
       format: "logger:stdout?json=true"
@@ -495,6 +372,9 @@ network_policy:
       - {}
 
 pod:
+  tolerations:
+    fluentd:
+      enabled: false
   security_context:
     fluentd:
       pod:

tools/deployment/common/fluentd-daemonset.sh

@@ -0,0 +1,179 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Lint and package chart
+make fluentd
+
+tee /tmp/fluentd-daemonset.yaml << EOF
+endpoints:
+  fluentd:
+    hosts:
+      default: fluentd-daemonset
+  prometheus_fluentd_exporter:
+    hosts:
+      default: fluentd-daemonset-exporter
+monitoring:
+  prometheus:
+    enabled: true
+pod:
+  security_context:
+    fluentd:
+      pod:
+        runAsUser: 0
+
+deployment:
+  type: DaemonSet
+conf:
+  fluentd:
+    template: |
+      <source>
+        bind 0.0.0.0
+        port 24220
+        @type monitor_agent
+      </source>
+
+      <source>
+        <parse>
+          time_format %Y-%m-%dT%H:%M:%S.%NZ
+          @type json
+        </parse>
+        path /var/log/containers/*.log
+        read_from_head true
+        tag kubernetes.*
+        @type tail
+      </source>
+
+      <filter kubernetes.**>
+        @type kubernetes_metadata
+      </filter>
+
+      <source>
+        bind 0.0.0.0
+        port "#{ENV['FLUENTD_PORT']}"
+        @type forward
+      </source>
+
+      <match fluent.**>
+        @type null
+      </match>
+
+      <match libvirt>
+        <buffer>
+          chunk_limit_size 500K
+          flush_interval 5s
+          flush_thread_count 8
+          queue_limit_length 16
+          retry_forever false
+          retry_max_interval 30
+        </buffer>
+        host "#{ENV['ELASTICSEARCH_HOST']}"
+        include_tag_key true
+        logstash_format true
+        logstash_prefix libvirt
+        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
+        port "#{ENV['ELASTICSEARCH_PORT']}"
+        @type elasticsearch
+        user "#{ENV['ELASTICSEARCH_USERNAME']}"
+      </match>
+
+      <match qemu>
+        <buffer>
+          chunk_limit_size 500K
+          flush_interval 5s
+          flush_thread_count 8
+          queue_limit_length 16
+          retry_forever false
+          retry_max_interval 30
+        </buffer>
+        host "#{ENV['ELASTICSEARCH_HOST']}"
+        include_tag_key true
+        logstash_format true
+        logstash_prefix qemu
+        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
+        port "#{ENV['ELASTICSEARCH_PORT']}"
+        @type elasticsearch
+        user "#{ENV['ELASTICSEARCH_USERNAME']}"
+      </match>
+
+      <match journal.**>
+        <buffer>
+          chunk_limit_size 500K
+          flush_interval 5s
+          flush_thread_count 8
+          queue_limit_length 16
+          retry_forever false
+          retry_max_interval 30
+        </buffer>
+        host "#{ENV['ELASTICSEARCH_HOST']}"
+        include_tag_key true
+        logstash_format true
+        logstash_prefix journal
+        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
+        port "#{ENV['ELASTICSEARCH_PORT']}"
+        @type elasticsearch
+        user "#{ENV['ELASTICSEARCH_USERNAME']}"
+      </match>
+
+      <match kernel>
+        <buffer>
+          chunk_limit_size 500K
+          flush_interval 5s
+          flush_thread_count 8
+          queue_limit_length 16
+          retry_forever false
+          retry_max_interval 30
+        </buffer>
+        host "#{ENV['ELASTICSEARCH_HOST']}"
+        include_tag_key true
+        logstash_format true
+        logstash_prefix kernel
+        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
+        port "#{ENV['ELASTICSEARCH_PORT']}"
+        @type elasticsearch
+        user "#{ENV['ELASTICSEARCH_USERNAME']}"
+      </match>
+
+      <match **>
+        <buffer>
+          chunk_limit_size 500K
+          flush_interval 5s
+          flush_thread_count 8
+          queue_limit_length 16
+          retry_forever false
+          retry_max_interval 30
+        </buffer>
+        flush_interval 15s
+        host "#{ENV['ELASTICSEARCH_HOST']}"
+        include_tag_key true
+        logstash_format true
+        password "#{ENV['ELASTICSEARCH_PASSWORD']}"
+        port "#{ENV['ELASTICSEARCH_PORT']}"
+        @type elasticsearch
+        type_name fluent
+        user "#{ENV['ELASTICSEARCH_USERNAME']}"
+      </match>
+EOF
+helm upgrade --install fluentd-daemonset ./fluentd \
+    --namespace=osh-infra \
+    --values=/tmp/fluentd-daemonset.yaml
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh osh-infra
+
+#NOTE: Validate Deployment info
+helm status fluentd-daemonset

tools/deployment/multinode/130-fluentd-daemonset.sh

@@ -0,0 +1 @@
+../common/fluentd-daemonset.sh

tools/deployment/multinode/135-fluentd-deployment.sh

@@ -20,7 +20,9 @@ set -xe
 make fluentd
 
 if [ ! -d "/var/log/journal" ]; then
-tee /tmp/fluentd.yaml << EOF
+tee /tmp/fluentd-deployment.yaml << EOF
+deployment:
+  type: Deployment
 monitoring:
   prometheus:
     enabled: true
@@ -36,16 +38,24 @@ pod:
           - name: runlog
             mountPath: /run/log
 EOF
-helm upgrade --install fluentd ./fluentd \
-    --namespace=osh-infra \
-    --values=/tmp/fluentd.yaml
 else
-helm upgrade --install fluentd ./fluentd \
-    --namespace=osh-infra
+tee /tmp/fluentd-deployment.yaml << EOF
+deployment:
+  type: Deployment
+monitoring:
+  prometheus:
+    enabled: true
+pod:
+  replicas:
+    fluentd: 1
+EOF
 fi
+helm upgrade --install fluentd-deployment ./fluentd \
+    --namespace=osh-infra \
+    --values=/tmp/fluentd-deployment.yaml
 
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh osh-infra
 
 #NOTE: Validate Deployment info
-helm status fluentd
+helm status fluentd-deployment

tools/deployment/osh-infra-logging/065-fluentd-daemonset.sh

@@ -0,0 +1 @@
+../common/fluentd-daemonset.sh

zuul.d/jobs.yaml

@@ -61,7 +61,8 @@
         - ./tools/deployment/multinode/115-radosgw-osh-infra.sh
         - ./tools/deployment/multinode/120-elasticsearch.sh
         - ./tools/deployment/multinode/125-fluentbit.sh
-        - ./tools/deployment/multinode/130-fluentd.sh
+        - ./tools/deployment/multinode/130-fluentd-daemonset.sh
+        - ./tools/deployment/multinode/135-fluentd-deployment.sh
         - ./tools/deployment/multinode/140-kibana.sh
         - ./tools/deployment/multinode/600-grafana-selenium.sh
         - ./tools/deployment/multinode/610-nagios-selenium.sh
@@ -129,8 +130,9 @@
         - ./tools/deployment/osh-infra-logging/050-elasticsearch.sh
         - ./tools/deployment/osh-infra-logging/055-elasticsearch-ldap.sh
         - ./tools/deployment/osh-infra-logging/060-fluentbit.sh
-        - ./tools/deployment/osh-infra-logging/065-fluentd.sh
-        - ./tools/deployment/osh-infra-logging/070-kibana.sh
+        - ./tools/deployment/osh-infra-logging/065-fluentd-daemonset.sh
+        - ./tools/deployment/osh-infra-logging/070-fluentd-deployment.sh
+        - ./tools/deployment/osh-infra-logging/075-kibana.sh
         - ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh
 
 - job: