Fluentd: Add support for Kafka outputs

This updates the fluentd chart to add support for leveraging a
Kafka output. This required adding a kafka endpoint entry to the
chart's values.yaml, as well as the required template updates to
the fluentd deployment template and the addition of a secret for
kafka credentials

Depends-On: https://review.opendev.org/#/c/679297/

Change-Id: I80a487a0538f0b3704fb598da38c07feedaccb0e
Signed-off-by: Steve Wilkerson <sw5822@att.com>

fluentd/templates/deployment-fluentd.yaml

@@ -19,8 +19,11 @@ limitations under the License.
 
 {{- $mounts_fluentd := .Values.pod.mounts.fluentd.fluentd }}
 
-{{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "fluentd"  }}
+{{- $kafkaBroker := tuple "kafka" "public" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $kafkaBrokerPort := tuple "kafka" "public" "broker" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $kafkaBrokerURI := printf "%s:%s" $kafkaBroker $kafkaBrokerPort }}
 
+{{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "fluentd"  }}
 {{ tuple $envAll "fluentd" $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1beta1
@@ -156,6 +159,8 @@ spec:
               value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" | quote }}
             - name: ELASTICSEARCH_PORT
               value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: KAFKA_BROKER
+              value: {{ $kafkaBrokerURI }}
             - name: ELASTICSEARCH_USERNAME
               valueFrom:
                 secretKeyRef:
@@ -166,6 +171,18 @@ spec:
                 secretKeyRef:
                   name: {{ printf "%s-%s" $envAll.Release.Name "elasticsearch-user" | quote }}
                   key: ELASTICSEARCH_PASSWORD
+{{- if .Values.manifests.secret_kafka }}
+            - name: KAFKA_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ printf "%s-%s" $envAll.Release.Name "kafka-user" | quote }}
+                  key: KAFKA_USERNAME
+            - name: KAFKA_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ printf "%s-%s" $envAll.Release.Name "kafka-user" | quote }}
+                  key: KAFKA_PASSWORD
+{{- end }}
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp

fluentd/templates/secret-kafka-creds.yaml

@@ -0,0 +1,28 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_kafka }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ printf "%s-%s" $envAll.Release.Name "kafka-user" | quote }}
+type: Opaque
+data:
+  KAFKA_USERNAME: {{ .Values.endpoints.kafka.auth.admin.username | b64enc }}
+  KAFKA_PASSWORD: {{ .Values.endpoints.kafka.auth.admin.password | b64enc }}
+{{- end }}

fluentd/values.yaml

@@ -347,6 +347,25 @@ endpoints:
         default: 24224
       metrics:
         default: 24220
+  kafka:
+    namespace: null
+    name: kafka
+    auth:
+      admin:
+        username: admin
+        password: changeme
+    hosts:
+      default: kafka-broker
+      public: kafka
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: kafka
+    port:
+      broker:
+        default: 9092
   prometheus_fluentd_exporter:
     namespace: null
     hosts:
@@ -461,4 +480,5 @@ manifests:
       service_exporter: true
   network_policy: false
   secret_elasticsearch: true
+  secret_kafka: false
   service_fluentd: true