Ceph: Remove fluentbit sidecars, mount hostpath for logs

This removes the fluentbit sidecars from the ceph-mon and ceph-osd
charts. Instead, we mount /var/log/ceph as a hostpath, and use the
fluentbit daemonset to target the mounted log files instead

This also updates the fluentd configuration to better handle the
correct configuration type for flush_interval (time vs int), as
well as updates the fluentd elasticsearch output values to help
address the gate failures resulting from the Elasticsearch bulk
endpoints failing

Change-Id: If3f2ff6371f267ed72379de25ff463079ba4cddc

ceph-mon/templates/bin/mon/_fluentbit-sidecar.sh.tpl

@@ -1,19 +0,0 @@
-#!/bin/sh
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-set -ex
-
-exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf

ceph-mon/templates/configmap-bin.yaml

@@ -60,8 +60,4 @@ data:
   utils-checkPGs.sh: |
 {{ tuple "bin/utils/_checkPGs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 
-{{ if .Values.logging.fluentd }}
-  fluentbit-sidecar.sh: |
-{{ tuple "bin/mon/_fluentbit-sidecar.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{ end }}
 {{- end }}

ceph-mon/templates/configmap-etc.yaml

@@ -42,15 +42,6 @@ limitations under the License.
 {{- if empty .Values.conf.ceph.osd.public_network -}}
 {{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
 {{- end -}}
-
-{{- if not (has "fluentd_output" .Values.conf.fluentbit) -}}
-{{- $fluentd_host := tuple "fluentd" "internal" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
-{{- $fluentd_port := tuple "fluentd" "internal" "service" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-{{- $fluentd_output := dict "header" "output" "Name" "forward" "Match" "*" "Host" $fluentd_host "Port" $fluentd_port -}}
-{{- $_ := set .Values "__fluentbit_config" ( list $fluentd_output) -}}
-{{- $__fluentbit_config := append .Values.conf.fluentbit .Values.__fluentbit_config -}}
-{{- $_ := set .Values.conf "fluentbit" $__fluentbit_config -}}
-{{- end -}}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -59,12 +50,6 @@ metadata:
 data:
   ceph.conf: |
 {{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
-{{ if .Values.logging.fluentd }}
-  fluent-bit.conf: |
-{{ include "ceph-mon.utils.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
-  parsers.conf: |
-{{ include "ceph-mon.utils.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
-{{ end }}
 {{- end }}
 {{- end }}
 {{- end }}

ceph-mon/templates/daemonset-mon.yaml

@@ -91,6 +91,19 @@ spec:
             - name: pod-run
               mountPath: /run
               readOnly: false
+        - name: ceph-log-ownership
+{{ tuple $envAll "ceph_mon" | include "helm-toolkit.snippets.image" | indent 10 }}
+          securityContext:
+            runAsUser: 0
+          command:
+            - chown
+            - -R
+            - ceph:root
+            - /var/log/ceph
+          volumeMounts:
+            - name: pod-var-log
+              mountPath: /var/log/ceph
+              readOnly: false
       containers:
         - name: ceph-mon
 {{ tuple $envAll "ceph_mon" | include "helm-toolkit.snippets.image" | indent 10 }}
@@ -195,33 +208,13 @@ spec:
             - name: pod-run
               mountPath: /run
               readOnly: false
-            - name: varlog
+            - name: pod-var-log
               mountPath: /var/log/ceph
-        {{ if .Values.logging.fluentd }}
+              readOnly: false
-        - name: fluentbit-sidecar
-{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
-{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-          command:
-            - /tmp/fluentbit-sidecar.sh
-          volumeMounts:
-            - name: ceph-mon-bin
-              mountPath: /tmp/fluentbit-sidecar.sh
-              subPath: fluentbit-sidecar.sh
-              readOnly: true
-            - name: varlog
-              mountPath: /var/log/ceph
-            - name: ceph-mon-etc
-              mountPath: /fluent-bit/etc/fluent-bit.conf
-              subPath: fluent-bit.conf
-              readOnly: true
-            - name: ceph-mon-etc
-              mountPath: /fluent-bit/etc/parsers.conf
-              subPath: parsers.conf
-              readOnly: true
-        {{ end }}
       volumes:
-        - name: varlog
+        - name: pod-var-log
-          emptyDir: {}
+          hostPath:
+            path: /var/log/ceph
         - name: ceph-mon-bin
           configMap:
             name: ceph-mon-bin

ceph-mon/templates/utils/_to_fluentbit_conf.tpl

@@ -1,38 +0,0 @@
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-# This function generates fluentbit configuration files with entries in the
-# ceph-mon values.yaml.  It results in a configuration section with the
-# following format (for as many key/value pairs defined in values for a section):
-# [HEADER]
-#     key value
-#     key value
-#     key value
-# The configuration schema can be found here:
-# http://fluentbit.io/documentation/0.12/configuration/schema.html
-
-{{- define "ceph-mon.utils.to_fluentbit_conf" -}}
-{{- range $values := . -}}
-{{- range $section := . -}}
-{{- $header := pick . "header" -}}
-{{- $config := omit . "header" }}
-[{{$header.header | upper }}]
-{{range $key, $value := $config -}}
-{{ $key | indent 4 }} {{ $value }}
-{{end -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}

ceph-mon/values.yaml

@@ -28,7 +28,6 @@ images:
     ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3'
     ceph_mon: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
     ceph_mon_check: 'docker.io/port/ceph-config-helper:v1.10.3'
-    fluentbit: docker.io/fluent/fluent-bit:0.12.14
     dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
     image_repo_sync: docker.io/docker:17.07.0
   local_registry:
@@ -71,13 +70,6 @@ pod:
       limits:
         memory: "50Mi"
         cpu: "500m"
-    fluentbit:
-      requests:
-        memory: "5Mi"
-        cpu: "250m"
-      limits:
-        memory: "50Mi"
-        cpu: "500m"
     jobs:
       bootstrap:
         limits:
@@ -187,40 +179,6 @@ conf:
   storage:
     mon:
       directory: /var/lib/openstack-helm/ceph/mon
-  fluentbit:
-    - service:
-        header: service
-        Flush: 30
-        Daemon: Off
-        Log_Level: info
-        Parsers_File: parsers.conf
-    - ceph_tail:
-        # NOTE(srwilkers): Despite being exposed, these values should not be
-        # modified, as the ceph-mon logs are always placed here
-        header: input
-        Name: tail
-        Tag: ceph-mon.*
-        Path: /var/log/ceph/*.log
-        Parser: syslog
-        DB: /var/log/ceph/ceph.db
-        DB.Sync: Normal
-        Buffer_Chunk_Size: 1M
-        Buffer_Max_Size: 1M
-        Mem_Buf_Limit: 5MB
-        Refresh_Interval: 10s
-  parsers:
-    - syslog:
-        header: parser
-        Name: syslog
-        Format: regex
-        Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
-        Time_Key: time
-        Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
-        Time_Keep: On
-        Types: "pid:integer"
-
-logging:
-  fluentd: false
 
 dependencies:
   dynamic:
@@ -320,22 +278,6 @@ endpoints:
     port:
       mon:
         default: 6789
-  fluentd:
-    namespace: null
-    name: fluentd
-    hosts:
-      default: fluentd-logging
-    host_fqdn_override:
-      default: null
-    path:
-      default: null
-    scheme:
-      default: http
-    port:
-      service:
-        default: 24224
-      metrics:
-        default: 24220
 
 manifests:
   configmap_bin: true

ceph-osd/templates/bin/osd/_fluentbit-sidecar.sh.tpl

@@ -1,20 +0,0 @@
-#!/bin/sh
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-set -ex
-
-exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf

ceph-osd/templates/configmap-bin.yaml

@@ -40,8 +40,4 @@ data:
 {{ tuple "bin/osd/_stop.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
   init-dirs.sh: |
 {{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{ if .Values.logging.fluentd }}
-  fluentbit-sidecar.sh: |
-{{ tuple "bin/osd/_fluentbit-sidecar.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{ end }}
 {{- end }}

ceph-osd/templates/configmap-etc.yaml

@@ -40,15 +40,6 @@ limitations under the License.
 {{- if empty .Values.conf.ceph.osd.public_network -}}
 {{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
 {{- end -}}
-
-{{- if not (has "fluentd_output" .Values.conf.fluentbit) -}}
-{{- $fluentd_host := tuple "fluentd" "internal" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
-{{- $fluentd_port := tuple "fluentd" "internal" "service" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-{{- $fluentd_output := dict "header" "output" "Name" "forward" "Match" "*" "Host" $fluentd_host "Port" $fluentd_port -}}
-{{- $_ := set .Values "__fluentbit_config" ( list $fluentd_output) -}}
-{{- $__fluentbit_config := append .Values.conf.fluentbit .Values.__fluentbit_config -}}
-{{- $_ := set .Values.conf "fluentbit" $__fluentbit_config -}}
-{{- end -}}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -57,12 +48,6 @@ metadata:
 data:
   ceph.conf: |
 {{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
-{{ if .Values.logging.fluentd }}
-  fluent-bit.conf: |
-{{ include "ceph-osd.utils.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
-  parsers.conf: |
-{{ include "ceph-osd.utils.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
-{{ end }}
 {{- end }}
 {{- end }}
 {{- if .Values.manifests.configmap_etc }}

ceph-osd/templates/daemonset-osd.yaml

@@ -75,6 +75,35 @@ spec:
             - name: pod-run
               mountPath: /run
               readOnly: false
+        - name: ceph-log-ownership
+{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
+          securityContext:
+            runAsUser: 0
+          env:
+          # NOTE(portdirect): These environment variables will be populated
+          # dynamicly at the point of render and added to all containers in the
+          # pod
+          # - name: JOURNAL_LOCATION
+          #   value: /var/lib/openstack-helm/ceph/osd/journal-one
+          # - name: STORAGE_LOCATION
+          #   value: /var/lib/openstack-helm/ceph/osd/data-one
+          # - name: JOURNAL_TYPE
+          #   value: directory
+          # - name: STORAGE_TYPE
+          #   value: directory
+            - name: CLUSTER
+              value: "ceph"
+            - name: CEPH_GET_ADMIN_KEY
+              value: "1"
+          command:
+            - chown
+            - -R
+            - ceph:root
+            - /var/log/ceph
+          volumeMounts:
+            - name: pod-var-log
+              mountPath: /var/log/ceph
+              readOnly: false
         - name: osd-init
 {{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.osd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@@ -218,30 +247,9 @@ spec:
             - name: journal
               mountPath: /var/lib/ceph/journal
               readOnly: false
-            - name: varlog
+            - name: pod-var-log
               mountPath: /var/log/ceph
-        {{ if .Values.logging.fluentd }}
+              readOnly: false
-        - name: fluentbit-sidecar
-{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
-{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-          command:
-            - /tmp/fluentbit-sidecar.sh
-          volumeMounts:
-            - name: ceph-osd-bin
-              mountPath: /tmp/fluentbit-sidecar.sh
-              subPath: fluentbit-sidecar.sh
-              readOnly: true
-            - name: varlog
-              mountPath: /var/log/ceph
-            - name: ceph-osd-etc
-              mountPath: /fluent-bit/etc/fluent-bit.conf
-              subPath: fluent-bit.conf
-              readOnly: true
-            - name: ceph-osd-etc
-              mountPath: /fluent-bit/etc/parsers.conf
-              subPath: parsers.conf
-              readOnly: true
-        {{ end }}
       volumes:
         - name: devices
           hostPath:
@@ -251,8 +259,9 @@ spec:
             path: /run/lvm
         - name: pod-var-lib-ceph
           emptyDir: {}
-        - name: varlog
+        - name: pod-var-log
-          emptyDir: {}
+          hostPath:
+            path: /var/log/ceph
         - name: pod-run
           emptyDir:
             medium: "Memory"

ceph-osd/templates/utils/_to_fluentbit_conf.tpl

@@ -1,38 +0,0 @@
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-# This function generates fluentbit configuration files with entries in the
-# ceph-osd values.yaml.  It results in a configuration section with the
-# following format (for as many key/value pairs defined in values for a section):
-# [HEADER]
-#     key value
-#     key value
-#     key value
-# The configuration schema can be found here:
-# http://fluentbit.io/documentation/0.12/configuration/schema.html
-
-{{- define "ceph-osd.utils.to_fluentbit_conf" -}}
-{{- range $values := . -}}
-{{- range $section := . -}}
-{{- $header := pick . "header" -}}
-{{- $config := omit . "header" }}
-[{{$header.header | upper }}]
-{{range $key, $value := $config -}}
-{{ $key | indent 4 }} {{ $value }}
-{{end -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}

ceph-osd/values.yaml

@@ -22,7 +22,6 @@ images:
   tags:
     ceph_osd: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
     ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
-    fluentbit: docker.io/fluent/fluent-bit:0.12.14
     dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
     image_repo_sync: docker.io/docker:17.07.0
   local_registry:
@@ -56,13 +55,6 @@ pod:
       limits:
         memory: "1024Mi"
         cpu: "1000m"
-    fluentbit:
-      requests:
-        memory: "5Mi"
-        cpu: "250m"
-      limits:
-        memory: "50Mi"
-        cpu: "500m"
     jobs:
       image_repo_sync:
         requests:
@@ -148,40 +140,6 @@ conf:
 #                 journal:
 #                   type: directory
 #                   location: /var/lib/openstack-helm/ceph/osd/journal-three
-  fluentbit:
-    - service:
-        header: service
-        Flush: 30
-        Daemon: Off
-        Log_Level: info
-        Parsers_File: parsers.conf
-    - ceph_tail:
-        # NOTE(srwilkers): Despite being exposed, these values should not be
-        # modified, as the ceph-osd logs are always placed here
-        header: input
-        Name: tail
-        Tag: ceph-osd.*
-        Path: /var/log/ceph/ceph-osd.**.log
-        Parser: syslog
-        DB: /var/log/ceph/ceph-osd.db
-        DB.Sync: Normal
-        Buffer_Chunk_Size: 1M
-        Buffer_Max_Size: 1M
-        Mem_Buf_Limit: 5MB
-        Refresh_Interval: 10s
-  parsers:
-    - syslog:
-        header: parser
-        Name: syslog
-        Format: regex
-        Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
-        Time_Key: time
-        Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
-        Time_Keep: On
-        Types: "pid:integer"
-
-logging:
-  fluentd: false
 
 dependencies:
   dynamic:
@@ -242,22 +200,6 @@ endpoints:
     port:
       mon:
         default: 6789
-  fluentd:
-    namespace: null
-    name: fluentd
-    hosts:
-      default: fluentd-logging
-    host_fqdn_override:
-      default: null
-    path:
-      default: null
-    scheme:
-      default: http
-    port:
-      service:
-        default: 24224
-      metrics:
-        default: 24220
 
 manifests:
   configmap_bin: true

elasticsearch/values.yaml

@@ -106,8 +106,8 @@ pod:
           default: kubernetes.io/hostname
   replicas:
     master: 3
-    data: 3
+    data: 2
-    client: 2
+    client: 3
   lifecycle:
     upgrades:
       deployments:

fluent-logging/templates/utils/_to_fluentd_conf.tpl

@@ -62,6 +62,8 @@ section):
 {{ $type | indent 4 }} {{ $innerValue }}
 {{- else if contains "ENV" ($innerValue | quote) }}
 {{ $innerKey | indent 4 }} {{ $innerValue | quote }}
+{{- else if eq $innerKey "flush_interval" }}
+{{ $innerKey | indent 4 }} {{ printf "%ss" $innerValue }}
 {{- else }}
 {{ $innerKey | indent 4 }} {{ $innerValue }}
 {{- end }}
@@ -75,6 +77,8 @@ section):
 {{ $type | indent 2 }} {{ $value }}
 {{- else if contains "ENV" ($value | quote) }}
 {{ $key | indent 2 }} {{ $value | quote }}
+{{- else if eq $key "flush_interval" }}
+{{ $key | indent 2 }} {{ printf "%ss" $value }}
 {{- else }}
 {{ $key | indent 2 }} {{ $value }}
 {{- end -}}

fluent-logging/values.yaml

@@ -301,9 +301,10 @@ conf:
         port: "#{ENV['ELASTICSEARCH_PORT']}"
         logstash_format: true
         logstash_prefix: journal
-        buffer_chunk_limit: 10M
+        buffer_chunk_limit: 2M
-        buffer_queue_limit: 32
+        buffer_queue_limit: 8
-        flush_interval: 20
+        flush_interval: "5"
+        flush_thread_count: 2
         max_retry_wait: 300
         disable_retry_limit: ""
         num_threads: 8
@@ -318,9 +319,10 @@ conf:
         port: "#{ENV['ELASTICSEARCH_PORT']}"
         logstash_format: true
         logstash_prefix: kernel
-        buffer_chunk_limit: 10M
+        buffer_chunk_limit: 2M
-        buffer_queue_limit: 32
+        buffer_queue_limit: 8
-        flush_interval: 20
+        flush_interval: "5"
+        flush_thread_count: 2
         max_retry_wait: 300
         disable_retry_limit: ""
         num_threads: 8
@@ -337,9 +339,10 @@ conf:
         host: "#{ENV['ELASTICSEARCH_HOST']}"
         port: "#{ENV['ELASTICSEARCH_PORT']}"
         logstash_format: true
-        buffer_chunk_limit: 10M
+        buffer_chunk_limit: 2M
-        buffer_queue_limit: 32
+        buffer_queue_limit: 8
-        flush_interval: 20
+        flush_interval: "5"
+        flush_thread_count: 2
         max_retry_wait: 300
         disable_retry_limit: ""
         num_threads: 8