Merge "Add fluentbit sidecars to ceph mon and ceph osd charts"

ceph-mon/templates/bin/mon/_fluentbit-sidecar.sh.tpl

@@ -0,0 +1,19 @@
+#!/bin/sh
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+set -ex
+
+exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf

ceph-mon/templates/configmap-bin.yaml

@@ -54,4 +54,8 @@ data:
   moncheck-reap-zombies.py: |
 {{ tuple "bin/moncheck/_reap-zombies.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 
+{{ if .Values.logging.fluentd }}
+  fluentbit-sidecar.sh: |
+{{ tuple "bin/mon/_fluentbit-sidecar.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ end }}
 {{- end }}

ceph-mon/templates/configmap-etc.yaml

@@ -38,6 +38,14 @@ limitations under the License.
 {{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
 {{- end -}}
 
+{{- if not (has "fluentd_output" .Values.conf.fluentbit) -}}
+{{- $fluentd_host := tuple "fluentd" "internal" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
+{{- $fluentd_port := tuple "fluentd" "internal" "service" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $fluentd_output := dict "header" "output" "Name" "forward" "Match" "*" "Host" $fluentd_host "Port" $fluentd_port -}}
+{{- $_ := set .Values "__fluentbit_config" ( list $fluentd_output) -}}
+{{- $__fluentbit_config := append .Values.conf.fluentbit .Values.__fluentbit_config -}}
+{{- $_ := set .Values.conf "fluentbit" $__fluentbit_config -}}
+{{- end -}}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -46,7 +54,12 @@ metadata:
 data:
   ceph.conf: |
 {{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
-
+{{ if .Values.logging.fluentd }}
+  fluent-bit.conf: |
+{{ include "ceph-mon.utils.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
+  parsers.conf: |
+{{ include "ceph-mon.utils.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
+{{ end }}
 {{- end }}
 {{- end }}
 {{- end }}

ceph-mon/templates/daemonset-mon.yaml

@@ -184,7 +184,33 @@ spec:
             - name: pod-run
               mountPath: /run
               readOnly: false
+            - name: varlog
+              mountPath: /var/log/ceph
+        {{ if .Values.logging.fluentd }}
+        - name: fluentbit-sidecar
+{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/fluentbit-sidecar.sh
+          volumeMounts:
+            - name: ceph-mon-bin
+              mountPath: /tmp/fluentbit-sidecar.sh
+              subPath: fluentbit-sidecar.sh
+              readOnly: true
+            - name: varlog
+              mountPath: /var/log/ceph
+            - name: ceph-mon-etc
+              mountPath: /fluent-bit/etc/fluent-bit.conf
+              subPath: fluent-bit.conf
+              readOnly: true
+            - name: ceph-mon-etc
+              mountPath: /fluent-bit/etc/parsers.conf
+              subPath: parsers.conf
+              readOnly: true
+        {{ end }}
       volumes:
+        - name: varlog
+          emptyDir: {}
         - name: ceph-mon-bin
           configMap:
             name: ceph-mon-bin

ceph-mon/templates/utils/_to_fluentbit_conf.tpl

@@ -0,0 +1,38 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+# This function generates fluentbit configuration files with entries in the
+# ceph-mon values.yaml.  It results in a configuration section with the
+# following format (for as many key/value pairs defined in values for a section):
+# [HEADER]
+#     key value
+#     key value
+#     key value
+# The configuration schema can be found here:
+# http://fluentbit.io/documentation/0.12/configuration/schema.html
+
+{{- define "ceph-mon.utils.to_fluentbit_conf" -}}
+{{- range $values := . -}}
+{{- range $section := . -}}
+{{- $header := pick . "header" -}}
+{{- $config := omit . "header" }}
+[{{$header.header | upper }}]
+{{range $key, $value := $config -}}
+{{ $key | indent 4 }} {{ $value }}
+{{end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}

ceph-mon/values.yaml

@@ -23,6 +23,7 @@ images:
     ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3'
     ceph_mon: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
     ceph_mon_check: 'docker.io/port/ceph-config-helper:v1.10.3'
+    fluentbit: docker.io/fluent/fluent-bit:0.12.14
     dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
     image_repo_sync: docker.io/docker:17.07.0
   local_registry:
@@ -65,6 +66,13 @@ pod:
       limits:
         memory: "50Mi"
         cpu: "500m"
+    fluentbit:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
     jobs:
       bootstrap:
         limits:
@@ -151,6 +159,37 @@ conf:
   storage:
     mon:
       directory: /var/lib/openstack-helm/ceph/mon
+  fluentbit:
+    - service:
+        header: service
+        Flush: 30
+        Daemon: Off
+        Log_Level: info
+        Parsers_File: parsers.conf
+    - ceph_tail:
+        # NOTE(srwilkers): Despite being exposed, these values should not be
+        # modified, as the ceph-mon logs are always placed here
+        header: input
+        Name: tail
+        Tag: ceph-mon.*
+        Path: /var/log/ceph/*.log
+        Parser: syslog
+        DB: /var/log/ceph/ceph.db
+        Mem_Buf_Limit: 5MB
+        Refresh_Interval: 10s
+  parsers:
+    - syslog:
+        header: parser
+        Name: syslog
+        Format: regex
+        Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
+        Time_Key: time
+        Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
+        Time_Keep: On
+        Types: "pid:integer"
+
+logging:
+  fluentd: false
 
 dependencies:
   dynamic:
@@ -250,6 +289,22 @@ endpoints:
     port:
       mon:
         default: 6789
+  fluentd:
+    namespace: null
+    name: fluentd
+    hosts:
+      default: fluentd-logging
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      service:
+        default: 24224
+      metrics:
+        default: 24220
 
 manifests:
   configmap_bin: true

ceph-osd/templates/bin/osd/_fluentbit-sidecar.sh.tpl

@@ -0,0 +1,20 @@
+#!/bin/sh
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf

ceph-osd/templates/configmap-bin.yaml

@@ -40,4 +40,8 @@ data:
 {{ tuple "bin/osd/_stop.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
   init-dirs.sh: |
 {{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ if .Values.logging.fluentd }}
+  fluentbit-sidecar.sh: |
+{{ tuple "bin/osd/_fluentbit-sidecar.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ end }}
 {{- end }}

ceph-osd/templates/configmap-etc.yaml

@@ -36,6 +36,14 @@ limitations under the License.
 {{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
 {{- end -}}
 
+{{- if not (has "fluentd_output" .Values.conf.fluentbit) -}}
+{{- $fluentd_host := tuple "fluentd" "internal" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
+{{- $fluentd_port := tuple "fluentd" "internal" "service" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $fluentd_output := dict "header" "output" "Name" "forward" "Match" "*" "Host" $fluentd_host "Port" $fluentd_port -}}
+{{- $_ := set .Values "__fluentbit_config" ( list $fluentd_output) -}}
+{{- $__fluentbit_config := append .Values.conf.fluentbit .Values.__fluentbit_config -}}
+{{- $_ := set .Values.conf "fluentbit" $__fluentbit_config -}}
+{{- end -}}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -44,7 +52,12 @@ metadata:
 data:
   ceph.conf: |
 {{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
-
+{{ if .Values.logging.fluentd }}
+  fluent-bit.conf: |
+{{ include "ceph-osd.utils.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
+  parsers.conf: |
+{{ include "ceph-osd.utils.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
+{{ end }}
 {{- end }}
 {{- end }}
 {{- if .Values.manifests.configmap_etc }}

ceph-osd/templates/daemonset-osd.yaml

@@ -80,7 +80,8 @@ spec:
             privileged: true
           env:
           # NOTE(portdirect): These environment variables will be populated
-          # dynamicly at the point of render.
+          # dynamicly at the point of render and added to all containers in the
+          # pod
           # - name: JOURNAL_LOCATION
           #   value: /var/lib/openstack-helm/ceph/osd/journal-one
           # - name: STORAGE_LOCATION
@@ -215,6 +216,30 @@ spec:
             - name: journal
               mountPath: /var/lib/ceph/journal
               readOnly: false
+            - name: varlog
+              mountPath: /var/log/ceph
+        {{ if .Values.logging.fluentd }}
+        - name: fluentbit-sidecar
+{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/fluentbit-sidecar.sh
+          volumeMounts:
+            - name: ceph-osd-bin
+              mountPath: /tmp/fluentbit-sidecar.sh
+              subPath: fluentbit-sidecar.sh
+              readOnly: true
+            - name: varlog
+              mountPath: /var/log/ceph
+            - name: ceph-osd-etc
+              mountPath: /fluent-bit/etc/fluent-bit.conf
+              subPath: fluent-bit.conf
+              readOnly: true
+            - name: ceph-osd-etc
+              mountPath: /fluent-bit/etc/parsers.conf
+              subPath: parsers.conf
+              readOnly: true
+        {{ end }}
       volumes:
         - name: devices
           hostPath:
@@ -224,6 +249,8 @@ spec:
             path: /run/lvm
         - name: pod-var-lib-ceph
           emptyDir: {}
+        - name: varlog
+          emptyDir: {}
         - name: pod-run
           emptyDir:
             medium: "Memory"

ceph-osd/templates/utils/_osd_daemonset_overrides.tpl

@@ -217,12 +217,16 @@ limitations under the License.
     {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }}
     {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }}
 
-    {{/* set container name
+    {{/* set container names and add to the list of containers for the pod */}}
-    assume not more than one container is defined */}}
+    {{- $_ := set $context.Values "__containers_list" ( list ) }}
-    {{- $container := first $context.Values.__daemonset_yaml.spec.template.spec.containers }}
+    {{- range $container := $context.Values.__daemonset_yaml.spec.template.spec.containers }}
+    {{- if eq $container.name "osd-pod" }}
     {{- $_ := set $container "name" $current_dict.dns_1123_name }}
-    {{- $cont_list := list $container }}
+    {{- end }}
-    {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "containers" $cont_list }}
+    {{- $__containers_list := append $context.Values.__containers_list $container }}
+    {{- $_ := set $context.Values "__containers_list" $__containers_list }}
+    {{- end }}
+    {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "containers" $context.Values.__containers_list }}
 
     {{/* cross-reference configmap name to container volume definitions */}}
     {{- $_ := set $context.Values "__volume_list" list }}
@@ -310,6 +314,9 @@ limitations under the License.
   {{- $_ := set $context.Values "__tmpYAMLcontainers" list }}
   {{- range $podContainer := $context.Values.__daemonset_yaml.spec.template.spec.containers }}
     {{- $_ := set $context.Values "_tmpYAMLcontainer" $podContainer }}
+    {{- if empty $context.Values._tmpYAMLcontainer.env }}
+    {{- $_ := set $context.Values._tmpYAMLcontainer "env" ( list ) }}
+    {{- end }}
     {{ $containerEnv := prepend (prepend (prepend ( prepend (index $context.Values._tmpYAMLcontainer "env") (dict "name" "STORAGE_TYPE" "value" $v.data.type)) (dict "name" "JOURNAL_TYPE" "value" $v.journal.type)) (dict "name" "STORAGE_LOCATION" "value" $v.data.location)) (dict "name" "JOURNAL_LOCATION" "value" $v.journal.location) }}
     {{- $localInitContainerEnv := omit $context.Values._tmpYAMLcontainer "env" }}
     {{- $_ := set $localInitContainerEnv "env" $containerEnv }}

ceph-osd/templates/utils/_to_fluentbit_conf.tpl

@@ -0,0 +1,38 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+# This function generates fluentbit configuration files with entries in the
+# ceph-osd values.yaml.  It results in a configuration section with the
+# following format (for as many key/value pairs defined in values for a section):
+# [HEADER]
+#     key value
+#     key value
+#     key value
+# The configuration schema can be found here:
+# http://fluentbit.io/documentation/0.12/configuration/schema.html
+
+{{- define "ceph-osd.utils.to_fluentbit_conf" -}}
+{{- range $values := . -}}
+{{- range $section := . -}}
+{{- $header := pick . "header" -}}
+{{- $config := omit . "header" }}
+[{{$header.header | upper }}]
+{{range $key, $value := $config -}}
+{{ $key | indent 4 }} {{ $value }}
+{{end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}

ceph-osd/values.yaml

@@ -17,6 +17,7 @@ images:
   tags:
     ceph_osd: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
     ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
+    fluentbit: docker.io/fluent/fluent-bit:0.12.14
     dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
     image_repo_sync: docker.io/docker:17.07.0
   local_registry:
@@ -50,6 +51,13 @@ pod:
       limits:
         memory: "1024Mi"
         cpu: "1000m"
+    fluentbit:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
     jobs:
       image_repo_sync:
         requests:
@@ -122,6 +130,37 @@ conf:
 #                 journal:
 #                   type: directory
 #                   location: /var/lib/openstack-helm/ceph/osd/journal-three
+  fluentbit:
+    - service:
+        header: service
+        Flush: 30
+        Daemon: Off
+        Log_Level: info
+        Parsers_File: parsers.conf
+    - ceph_tail:
+        # NOTE(srwilkers): Despite being exposed, these values should not be
+        # modified, as the ceph-osd logs are always placed here
+        header: input
+        Name: tail
+        Tag: ceph-osd.*
+        Path: /var/log/ceph/ceph-osd.**.log
+        Parser: syslog
+        DB: /var/log/ceph/ceph-osd.db
+        Mem_Buf_Limit: 5MB
+        Refresh_Interval: 10s
+  parsers:
+    - syslog:
+        header: parser
+        Name: syslog
+        Format: regex
+        Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
+        Time_Key: time
+        Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
+        Time_Keep: On
+        Types: "pid:integer"
+
+logging:
+  fluentd: false
 
 dependencies:
   dynamic:
@@ -182,6 +221,22 @@ endpoints:
     port:
       mon:
         default: 6789
+  fluentd:
+    namespace: null
+    name: fluentd
+    hosts:
+      default: fluentd-logging
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      service:
+        default: 24224
+      metrics:
+        default: 24220
 
 manifests:
   configmap_bin: true

tools/deployment/armada/multinode/armada-ceph.yaml

@@ -73,6 +73,8 @@ data:
         namespace: ceph
       ceph_mon:
         namespace: ceph
+      fluentd:
+        namespace: osh-infra
     network:
       public: ${CEPH_NETWORK}
       cluster: ${CEPH_NETWORK}
@@ -109,6 +111,9 @@ data:
             journal:
               type: directory
               location: /var/lib/openstack-helm/ceph/osd/journal-one
+    logging:
+      fluentd:
+        enabled: true
   source:
     type: local
     location: ${OSH_PATH}
@@ -141,6 +146,8 @@ data:
         namespace: ceph
       ceph_mon:
         namespace: ceph
+      fluentd:
+        namespace: osh-infra
     network:
       public: ${CEPH_NETWORK}
       cluster: ${CEPH_NETWORK}
@@ -177,6 +184,9 @@ data:
             journal:
               type: directory
               location: /var/lib/openstack-helm/ceph/osd/journal-one
+    logging:
+      fluentd:
+        enabled: true
   source:
     type: local
     location: ${OSH_PATH}