Separate fluentbit and fluentd charts

This begins to split the fluent-logging chart into two separate
charts, one for fluentbit and one for fluentd. This is to help
isolate each chart and its dependencies better, and to treat each
service as its own entity.

This also moves the job for creating Elasticsearch templates to
the Elasticsearch chart, as the elasticsearch chart should have
ownership of creating the templates for its indices.

This also performs some general cleanup of values keys that are
not currently used

Change-Id: I827277d5faa62b8b59c5960330703d23c297ca47
Signed-off-by: Steve Wilkerson <sw5822@att.com>

doc/source/install/multinode.rst

@@ -210,10 +210,10 @@ Alternatively, this step can be performed by running the script directly:
 
   ./tools/deployment/multinode/120-elasticsearch.sh
 
-Deploy Fluent-Logging
+Deploy Fluentbit
-^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^
 
-.. literalinclude:: ../../../tools/deployment/multinode/130-fluent-logging.sh
+.. literalinclude:: ../../../tools/deployment/multinode/125-fluentbit.sh
     :language: shell
     :lines: 1,17-
 
@@ -221,4 +221,17 @@ Alternatively, this step can be performed by running the script directly:
 
 .. code-block:: shell
 
-  ./tools/deployment/multinode/130-fluent-logging.sh
+  ./tools/deployment/multinode/125-fluentbit.sh
+
+Deploy Fluentd
+^^^^^^^^^^^^^^
+
+.. literalinclude:: ../../../tools/deployment/multinode/130-fluentd.sh
+    :language: shell
+    :lines: 1,17-
+
+Alternatively, this step can be performed by running the script directly:
+
+.. code-block:: shell
+
+  ./tools/deployment/multinode/130-fluentd.sh

elasticsearch/templates/bin/_helm-tests.sh.tpl

@@ -52,6 +52,24 @@ function check_snapshot_repositories () {
   {{ end }}
 }
 
+{{ if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }}
+# Tests whether elasticsearch has successfully generated the elasticsearch index mapping
+# templates defined by values.yaml
+function check_templates () {
+  {{ range $template, $fields := .Values.conf.templates }}
+  {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+              -XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \
+              | python -c "import sys, json; print len(json.load(sys.stdin))")
+  if [ "${{$template}}_total_hits" -gt 0 ]; then
+     echo "PASS: Successful hits on {{$template}} template!"
+  else
+     echo "FAIL: No hits on query for {{$template}} template! Exiting";
+     exit 1;
+  fi
+  {{ end }}
+}
+{{ end }}
+
 function remove_test_index () {
   echo "Deleting index created for service testing"
   curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
@@ -63,4 +81,5 @@ create_test_index
 {{ if .Values.conf.elasticsearch.snapshots.enabled }}
 check_snapshot_repositories
 {{ end }}
+check_templates
 remove_test_index

elasticsearch/templates/configmap-bin-elasticsearch.yaml

@@ -38,6 +38,8 @@ data:
 {{ tuple "bin/_register-repository.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   es-cluster-wait.sh: |
 {{ tuple "bin/_es-cluster-wait.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  create_template.sh: |
+{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   image-repo-sync.sh: |
 {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
 {{- end }}

elasticsearch/templates/configmap-etc-templates.yaml

@@ -1,11 +1,11 @@
 {{/*
-Copyright 2017-2018 The Openstack-Helm Authors.
+Copyright 2019 The Openstack-Helm Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+   http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +14,17 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and .Values.manifests.monitoring.prometheus.network_policy_exporter .Values.monitoring.prometheus.enabled -}}
+{{- if .Values.manifests.configmap_etc_templates }}
-{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-fluentd-exporter" }}
+{{- $envAll := . }}
-{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
+
-{{- end -}}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: elasticsearch-templates-etc
+type: Opaque
+data:
+{{ range $template, $fields := .Values.conf.templates }}
+  {{ $template }}.json: {{ toJson $fields | b64enc }}
+{{ end }}
+{{- end }}

elasticsearch/templates/job-elasticsearch-template.yaml

@@ -14,26 +14,26 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }}
+{{- if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }}
 {{- $envAll := . }}
 {{- $esUserSecret := .Values.secrets.elasticsearch.user }}
-{{- $mounts_elasticsearch_template := .Values.pod.mounts.elasticsearch_template.elasticsearch_template }}
+{{- $mounts_elasticsearch_templates := .Values.pod.mounts.elasticsearch_templates.elasticsearch_templates }}
-{{- $mounts_elasticsearch_template_init := .Values.pod.mounts.elasticsearch_template.init_container }}
+{{- $mounts_elasticsearch_templates_init := .Values.pod.mounts.elasticsearch_templates.init_container }}
 
-{{- $serviceAccountName := "fluent-logging-elasticsearch-template" }}
+{{- $serviceAccountName := "create-elasticsearch-templates" }}
-{{ tuple $envAll "elasticsearch_template" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+{{ tuple $envAll "elasticsearch_templates" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: elasticsearch-template
+  name: create-elasticsearch-templates
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
 spec:
   template:
     metadata:
       labels:
-{{ tuple $envAll "fluentd" "elasticsearch-template" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+{{ tuple $envAll "elasticsearch" "create-templates" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "elasticsearch_template" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}
@@ -41,11 +41,11 @@ spec:
       nodeSelector:
         {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }}
       initContainers:
-{{ tuple $envAll "elasticsearch_template" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+{{ tuple $envAll "elasticsearch_templates" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
-        - name: elasticsearch-template
+        - name: create-elasticsearch-templates
-{{ tuple $envAll "elasticsearch_template" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll "elasticsearch_templates" | include "helm-toolkit.snippets.image" | indent 10 }}
-{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_template | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_templates | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "elasticsearch_template" "container" "elasticsearch_template" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           env:
             - name: ELASTICSEARCH_HOST
@@ -67,27 +67,27 @@ spec:
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
-            - name: fluent-logging-bin
+            - name: elasticsearch-bin
               mountPath: /tmp/create_template.sh
               subPath: create_template.sh
               readOnly: true
             {{ range $template, $fields := .Values.conf.templates }}
-            - name: fluent-logging-etc
+            - name: elasticsearch-templates-etc
               mountPath: /tmp/{{$template}}.json
               subPath: {{$template}}.json
               readOnly: true
             {{ end }}
-{{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 12 }}{{ end }}
+{{ if $mounts_elasticsearch_templates.volumeMounts }}{{ toYaml $mounts_elasticsearch_templates.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: pod-tmp
           emptyDir: {}
-        - name: fluent-logging-bin
+        - name: elasticsearch-bin
           configMap:
-            name: fluent-logging-bin
+            name: elasticsearch-bin
             defaultMode: 0555
-        - name: fluent-logging-etc
+        - name: elasticsearch-templates-etc
           secret:
-            secretName: fluent-logging-etc
+            secretName: elasticsearch-templates-etc
             defaultMode: 0444
-{{ if $mounts_elasticsearch_template.volumes }}{{ toYaml $mounts_elasticsearch_template.volumes | indent 8 }}{{ end }}
+{{ if $mounts_elasticsearch_templates.volumes }}{{ toYaml $mounts_elasticsearch_templates.volumes | indent 8 }}{{ end }}
 {{- end }}

elasticsearch/values.yaml

@@ -30,6 +30,7 @@ images:
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
     snapshot_repository: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_xenial
     es_cluster_wait: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_xenial
+    elasticsearch_templates: docker.io/openstackhelm/heat:newton
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: "IfNotPresent"
   local_registry:
@@ -82,6 +83,10 @@ dependencies:
       services:
         - endpoint: internal
           service: elasticsearch
+    elasticsearch_templates:
+      services: null
+      jobs:
+        - elasticsearch-cluster-wait
     image_repo_sync:
       services:
         - endpoint: internal
@@ -220,6 +225,8 @@ pod:
   mounts:
     elasticsearch:
       elasticsearch:
+    elasticsearch_templates:
+      elasticsearch_templates:
   resources:
     enabled: false
     apache_proxy:
@@ -265,6 +272,13 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+      elasticsearch_templates:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
       image_repo_sync:
         requests:
           memory: "128Mi"
@@ -635,6 +649,35 @@ conf:
     es:
       all: true
       timeout: 20s
+  templates:
+    fluent:
+      template: "logstash-*"
+      index_patterns: "logstash-*"
+      settings:
+        number_of_shards: 1
+      mappings:
+        fluent:
+          properties:
+            kubernetes:
+              properties:
+                container_name:
+                  type: keyword
+                  index: false
+                docker_id:
+                  type: keyword
+                  index: false
+                host:
+                  type: keyword
+                  index: false
+                namespace_name:
+                  type: keyword
+                  index: false
+                pod_id:
+                  type: keyword
+                  index: false
+                pod_name:
+                  type: keyword
+                  index: false
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -771,11 +814,13 @@ manifests:
   configmap_bin_elasticsearch: true
   configmap_etc_curator: true
   configmap_etc_elasticsearch: true
+  configmap_etc_templates: true
   cron_curator: true
   deployment_client: true
   deployment_master: true
   ingress: true
   job_cluster_wait: true
+  job_elasticsearch_templates: true
   job_image_repo_sync: true
   job_snapshot_repository: true
   job_s3_user: true

fluent-logging/README.rst

@@ -1,30 +0,0 @@
-Fluentd-logging
-===============
-
-OpenStack-Helm defines a centralized logging mechanism to provide insight into
-the state of the OpenStack services and infrastructure components as
-well as underlying kubernetes platform. Among the requirements for a logging
-platform, where log data can come from and where log data need to be delivered
-are very variable. To support various logging scenarios, OpenStack-Helm should
-provide a flexible mechanism to meet with certain operation needs. This chart
-proposes fast and lightweight log forwarder and full featured log aggregator
-complementing each other providing a flexible and reliable solution. Especially,
-Fluent-bit is proposed as a log forwarder and Fluentd is proposed as a main log
-aggregator and processor.
-
-
-Mechanism
----------
-
-Fluent-bit, Fluentd meet OpenStack-Helm's logging requirements for gathering,
-aggregating, and delivering of logged events. Flunt-bit runs as a daemonset on
-each node and mounts the /var/lib/docker/containers directory. The Docker
-container runtime engine directs events posted to stdout and stderr to this
-directory on the host. Fluent-bit then forward the contents of that directory to
-Fluentd. Fluentd runs as deployment at the designated nodes and expose service
-for Fluent-bit to foward logs. Fluentd should then apply the Logstash format to
-the logs. Fluentd can also write kubernetes and OpenStack metadata to the logs.
-Fluentd will then forward the results to Elasticsearch and to optionally kafka.
-Elasticsearch indexes the logs in a logstash-* index by default. kafka stores
-the logs in a 'logs' topic by default. Any external tool can then consume the
-'logs' topic.

fluent-logging/templates/bin/_helm-tests.sh.tpl

@@ -1,41 +0,0 @@
-#!/bin/bash
-
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-set -ex
-
-{{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }}
-# Tests whether fluent-logging has successfully generated the elasticsearch index mapping
-# templates defined by values.yaml
-function check_templates () {
-  {{ range $template, $fields := .Values.conf.templates }}
-  {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-              -XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \
-              | python -c "import sys, json; print len(json.load(sys.stdin))")
-  if [ "${{$template}}_total_hits" -gt 0 ]; then
-     echo "PASS: Successful hits on {{$template}} template, provided by fluent-logging!"
-  else
-     echo "FAIL: No hits on query for {{$template}} template! Exiting";
-     exit 1;
-  fi
-  {{ end }}
-}
-{{ end }}
-
-{{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }}
-check_templates
-{{ end }}

fluent-logging/templates/pod-helm-tests.yaml

@@ -1,75 +0,0 @@
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- if .Values.manifests.helm_tests }}
-{{- $envAll := . }}
-{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
-
-{{- $serviceAccountName := print .Release.Name "-test" }}
-{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: "{{.Release.Name}}-test"
-  labels:
-{{ tuple $envAll "fluentd" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
-  annotations:
-    "helm.sh/hook": test-success
-    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
-spec:
-{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }}
-  serviceAccountName: {{ $serviceAccountName }}
-  nodeSelector:
-    {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }}
-  restartPolicy: Never
-  initContainers:
-{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }}
-  containers:
-    - name: {{.Release.Name}}-helm-tests
-{{ tuple $envAll "helm_tests" | include "helm-toolkit.snippets.image" | indent 6 }}
-{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }}
-{{ dict "envAll" $envAll "application" "test" "container" "helm_tests" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }}
-      command:
-        - /tmp/helm-tests.sh
-      env:
-        - name: ELASTICSEARCH_USERNAME
-          valueFrom:
-            secretKeyRef:
-              name: {{ $esUserSecret }}
-              key: ELASTICSEARCH_USERNAME
-        - name: ELASTICSEARCH_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: {{ $esUserSecret }}
-              key: ELASTICSEARCH_PASSWORD
-        - name: ELASTICSEARCH_ENDPOINT
-          value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
-      volumeMounts:
-        - name: pod-tmp
-          mountPath: /tmp
-        - name: fluent-logging-bin
-          mountPath: /tmp/helm-tests.sh
-          subPath: helm-tests.sh
-          readOnly: true
-  volumes:
-    - name: pod-tmp
-      emptyDir: {}
-    - name: fluent-logging-bin
-      configMap:
-        name: fluent-logging-bin
-        defaultMode: 0555
-{{- end }}

fluentbit/Chart.yaml

@@ -0,0 +1,24 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+description: OpenStack-Helm Fluentbit
+name: fluentbit
+version: 0.1.0
+home: http://www.fluentbit.io/
+sources:
+  - https://github.com/fluent/fluentbit
+  - https://git.openstack.org/cgit/openstack/openstack-helm-infra
+maintainers:
+  - name: OpenStack-Helm Authors

fluentbit/templates/configmap-bin.yaml

@@ -20,16 +20,10 @@ limitations under the License.
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: fluent-logging-bin
+  name: fluentbit-bin
 data:
-  fluentd.sh: |
-{{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   fluent-bit.sh: |
 {{ tuple "bin/_fluent-bit.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  helm-tests.sh: |
-{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  create_template.sh: |
-{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   image-repo-sync.sh: |
 {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
 {{- end }}

fluentbit/templates/configmap-etc.yaml

@@ -20,13 +20,9 @@ limitations under the License.
 apiVersion: v1
 kind: Secret
 metadata:
-  name: fluent-logging-etc
+  name: fluentbit-etc
 type: Opaque
 data:
   fluent-bit.conf: {{ .Values.conf.fluentbit.template | b64enc }}
   parsers.conf: {{ .Values.conf.parsers.template | b64enc }}
-  fluent.conf: {{ .Values.conf.fluentd.template | b64enc }}
-{{ range $template, $fields := .Values.conf.templates }}
-  {{ $template }}.json: {{ toJson $fields | b64enc }}
-{{ end }}
 {{- end }}

fluentbit/templates/daemonset-fluent-bit.yaml

@@ -95,7 +95,7 @@ spec:
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
 {{ dict "envAll" $envAll "podName" "fluentbit" "containerNames" (list "fluentbit") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
-{{ dict "envAll" $envAll "application" "daemon" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+{{ dict "envAll" $envAll "application" "fluentbit" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}
 {{ if $envAll.Values.pod.tolerations.fluentbit.enabled }}
 {{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
@@ -112,7 +112,7 @@ spec:
         - name: fluentbit
 {{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-{{ dict "envAll" $envAll "application" "daemon" "container" "fluentbit" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ dict "envAll" $envAll "application" "fluentbit" "container" "fluentbit" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           command:
             - /tmp/fluent-bit.sh
           env:
@@ -123,7 +123,7 @@ spec:
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
-            - name: fluent-logging-bin
+            - name: fluentbit-bin
               mountPath: /tmp/fluent-bit.sh
               subPath: fluent-bit.sh
               readOnly: true
@@ -133,11 +133,11 @@ spec:
             - name: varlibdockercontainers
               mountPath: /var/lib/docker/containers
               readOnly: true
-            - name: fluent-logging-etc
+            - name: fluentbit-etc
               mountPath: /fluent-bit/etc/fluent-bit.conf
               subPath: fluent-bit.conf
               readOnly: true
-            - name: fluent-logging-etc
+            - name: fluentbit-etc
               mountPath: /fluent-bit/etc/parsers.conf
               subPath: parsers.conf
               readOnly: true
@@ -151,13 +151,13 @@ spec:
         - name: varlibdockercontainers
           hostPath:
             path: /var/lib/docker/containers
-        - name: fluent-logging-bin
+        - name: fluentbit-bin
           configMap:
-            name: fluent-logging-bin
+            name: fluentbit-bin
             defaultMode: 0555
-        - name: fluent-logging-etc
+        - name: fluentbit-etc
           secret:
-            secretName: fluent-logging-etc
+            secretName: fluentbit-etc
             defaultMode: 0444
 {{ if $mounts_fluentbit.volumes }}{{ toYaml $mounts_fluentbit.volumes | indent 8 }}{{ end }}
 {{- end }}

fluentbit/templates/job-image-repo-sync.yaml

@@ -15,6 +15,6 @@ limitations under the License.
 */}}
 
 {{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }}
-{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluent-logging" -}}
+{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluentbit" -}}
 {{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }}
 {{- end }}

fluentbit/values.yaml

@@ -0,0 +1,256 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default values for fluentbit
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+release_group: null
+
+labels:
+  fluentbit:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+
+images:
+  tags:
+    fluentbit: docker.io/fluent/fluent-bit:0.14.2
+    dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
+    image_repo_sync: docker.io/docker:17.07.0
+  pull_policy: IfNotPresent
+  local_registry:
+    active: false
+    exclude:
+      - dep_check
+      - image_repo_sync
+
+dependencies:
+  dynamic:
+    common:
+      local_image_registry:
+        jobs:
+          - fluentbit-image-repo-sync
+        services:
+          - endpoint: node
+            service: local_image_registry
+  static:
+    image_repo_sync:
+      services:
+        - endpoint: internal
+          service: local_image_registry
+
+conf:
+  fluentbit:
+    template: |
+      [SERVICE]
+          Daemon false
+          Flush 30
+          Log_Level info
+          Parsers_File parsers.conf
+
+      [INPUT]
+          Buffer_Chunk_Size 1M
+          Buffer_Max_Size 1M
+          Mem_Buf_Limit 5MB
+          Name tail
+          Path /var/log/kern.log
+          Tag kernel
+
+      [INPUT]
+          Buffer_Chunk_Size 1M
+          Buffer_Max_Size 1M
+          Mem_Buf_Limit 5MB
+          Name tail
+          Parser docker
+          Path /var/log/containers/*.log
+          Tag kube.*
+
+      [INPUT]
+          Buffer_Chunk_Size 1M
+          Buffer_Max_Size 1M
+          Mem_Buf_Limit 5MB
+          Name tail
+          Path /var/log/libvirt/libvirtd.log
+          Tag libvirt
+
+      [INPUT]
+          Buffer_Chunk_Size 1M
+          Buffer_Max_Size 1M
+          Mem_Buf_Limit 5MB
+          Name tail
+          Path /var/log/libvirt/qemu/*.log
+          Tag qemu
+
+      [INPUT]
+          Buffer_Chunk_Size 1M
+          Buffer_Max_Size 1M
+          Mem_Buf_Limit 5MB
+          Name systemd
+          Path ${JOURNAL_PATH}
+          Systemd_Filter _SYSTEMD_UNIT=kubelet.service
+          Tag journal.*
+
+      [INPUT]
+          Buffer_Chunk_Size 1M
+          Buffer_Max_Size 1M
+          Mem_Buf_Limit 5MB
+          Name systemd
+          Path ${JOURNAL_PATH}
+          Systemd_Filter _SYSTEMD_UNIT=docker.service
+          Tag journal.*
+
+      [FILTER]
+          Interval 1s
+          Match **
+          Name throttle
+          Rate 1000
+          Window 300
+
+      [FILTER]
+          Match libvirt
+          Name record_modifier
+          Record hostname ${HOSTNAME}
+
+      [FILTER]
+          Match qemu
+          Name record_modifier
+          Record hostname ${HOSTNAME}
+
+      [FILTER]
+          Match kernel
+          Name record_modifier
+          Record hostname ${HOSTNAME}
+
+      [FILTER]
+          Match journal.**
+          Name modify
+          Rename _BOOT_ID BOOT_ID
+          Rename _CAP_EFFECTIVE CAP_EFFECTIVE
+          Rename _CMDLINE CMDLINE
+          Rename _COMM COMM
+          Rename _EXE EXE
+          Rename _GID GID
+          Rename _HOSTNAME HOSTNAME
+          Rename _MACHINE_ID MACHINE_ID
+          Rename _PID PID
+          Rename _SYSTEMD_CGROUP SYSTEMD_CGROUP
+          Rename _SYSTEMD_SLICE SYSTEMD_SLICE
+          Rename _SYSTEMD_UNIT SYSTEMD_UNIT
+          Rename _TRANSPORT TRANSPORT
+          Rename _UID UID
+
+      [OUTPUT]
+          Match **.fluentd**
+          Name null
+
+      [FILTER]
+          Match kube.*
+          Merge_JSON_Log true
+          Name kubernetes
+
+      [OUTPUT]
+          Host ${FLUENTD_HOST}
+          Match *
+          Name forward
+          Port ${FLUENTD_PORT}
+  parsers:
+    template: |
+      [PARSER]
+        Decode_Field_As escaped_utf8 log
+        Format json
+        Name docker
+        Time_Format %Y-%m-%dT%H:%M:%S.%L
+        Time_Keep true
+        Time_Key time
+
+endpoints:
+  cluster_domain_suffix: cluster.local
+  local_image_registry:
+    name: docker-registry
+    namespace: docker-registry
+    hosts:
+      default: localhost
+      internal: docker-registry
+      node: localhost
+    host_fqdn_override:
+      default: null
+    port:
+      registry:
+        node: 5000
+  fluentd:
+    namespace: null
+    name: fluentd
+    hosts:
+      default: fluentd-logging
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      service:
+        default: 24224
+      metrics:
+        default: 24220
+
+pod:
+  security_context:
+    fluentbit:
+      pod:
+        runAsUser: 65534
+      container:
+        fluentbit:
+          runAsUser: 0
+          readOnlyRootFilesystem: false
+  affinity:
+    anti:
+      type:
+        default: preferredDuringSchedulingIgnoredDuringExecution
+      topologyKey:
+        default: kubernetes.io/hostname
+  lifecycle:
+    upgrades:
+      daemonsets:
+        pod_replacement_strategy: RollingUpdate
+        fluentbit:
+          enabled: true
+          min_ready_seconds: 0
+          max_unavailable: 1
+  resources:
+    enabled: false
+    fluentbit:
+      limits:
+        memory: '400Mi'
+        cpu: '400m'
+      requests:
+        memory: '100Mi'
+        cpu: '100m'
+  tolerations:
+    fluentbit:
+      enabled: false
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      - key: node-role.kubernetes.io/node
+        operator: Exists
+  mounts:
+    fluentbit:
+      fluentbit:
+
+manifests:
+  configmap_bin: true
+  configmap_etc: true
+  daemonset_fluentbit: true
+  job_image_repo_sync: true

fluentd/Chart.yaml

@@ -14,11 +14,10 @@
 
 apiVersion: v1
 description: OpenStack-Helm Fluentd
-name: fluent-logging
+name: fluentd
 version: 0.1.0
 home: http://www.fluentbit.io/
 sources:
-  - https://github.com/fluent/fluentbit
   - https://github.com/fluent/fluentd
   - https://git.openstack.org/cgit/openstack/openstack-helm-infra
 maintainers:

fluentd/requirements.yaml

@@ -0,0 +1,18 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dependencies:
+  - name: helm-toolkit
+    repository: http://localhost:8879/charts/
+    version: 0.1.0

fluentd/templates/configmap-bin.yaml

@@ -0,0 +1,29 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_bin }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fluentd-bin
+data:
+  fluentd.sh: |
+{{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  image-repo-sync.sh: |
+{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
+{{- end }}

fluentd/templates/configmap-etc.yaml

@@ -0,0 +1,27 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_etc }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: fluentd-etc
+type: Opaque
+data:
+  fluent.conf: {{ .Values.conf.fluentd.template | b64enc }}
+{{- end }}

fluentd/templates/deployment-fluentd.yaml

@@ -95,7 +95,7 @@ spec:
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
     spec:
-{{ dict "envAll" $envAll "application" "internal" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+{{ dict "envAll" $envAll "application" "fluentd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}
       affinity:
 {{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
@@ -108,7 +108,7 @@ spec:
         - name: fluentd
 {{ tuple $envAll "fluentd" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.fluentd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-{{ dict "envAll" $envAll "application" "internal" "container" "fluentd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ dict "envAll" $envAll "application" "fluentd" "container" "fluentd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           command:
             - /tmp/fluentd.sh
             - start
@@ -149,11 +149,11 @@ spec:
               mountPath: /tmp
             - name: pod-etc-fluentd
               mountPath: /fluentd/etc
-            - name: fluent-logging-etc
+            - name: fluentd-etc
               mountPath: /fluentd/etc/fluent.conf
               subPath: fluent.conf
               readOnly: true
-            - name: fluent-logging-bin
+            - name: fluentd-bin
               mountPath: /tmp/fluentd.sh
               subPath: fluentd.sh
               readOnly: true
@@ -163,13 +163,13 @@ spec:
           emptyDir: {}
         - name: pod-etc-fluentd
           emptyDir: {}
-        - name: fluent-logging-etc
+        - name: fluentd-etc
           secret:
-            secretName: fluent-logging-etc
+            secretName: fluentd-etc
             defaultMode: 0444
-        - name: fluent-logging-bin
+        - name: fluentd-bin
           configMap:
-            name: fluent-logging-bin
+            name: fluentd-bin
             defaultMode: 0555
 {{- if $mounts_fluentd.volumes }}{{ toYaml $mounts_fluentd.volumes | indent 8 }}{{- end }}
 {{- end }}

fluentd/templates/job-image-repo-sync.yaml

@@ -0,0 +1,20 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }}
+{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluentd" -}}
+{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }}
+{{- end }}

fluentd/values.yaml

@@ -22,22 +22,12 @@ labels:
   fluentd:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
-  fluentbit:
-    node_selector_key: openstack-control-plane
-    node_selector_value: enabled
   prometheus_fluentd_exporter:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
-  job:
-    node_selector_key: openstack-control-plane
-    node_selector_value: enabled
-  test:
-    node_selector_key: openstack-control-plane
-    node_selector_value: enabled
 
 images:
   tags:
-    fluentbit: docker.io/fluent/fluent-bit:0.14.2
     fluentd: docker.io/fluent/fluentd-kubernetes-daemonset:v1.2-debian-elasticsearch
     prometheus_fluentd_exporter: docker.io/bitnami/fluentd-exporter:0.2.0
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
@@ -60,33 +50,13 @@ dependencies:
     common:
       local_image_registry:
         jobs:
-          - fluent-logging-image-repo-sync
+          - fluentd-image-repo-sync
         services:
           - endpoint: node
             service: local_image_registry
   static:
-    elasticsearch_template:
-      services:
-        - endpoint: internal
-          service: elasticsearch
-    fluentbit:
-      jobs:
-        - elasticsearch-template
-      services:
-        - endpoint: internal
-          service: fluentd
     fluentd:
-      jobs:
+      services: null
-        - elasticsearch-template
-      services:
-        - endpoint: internal
-          service: elasticsearch
-    fluentd_with_kafka:
-      services:
-        - endpoint: internal
-          service: elasticsearch
-        - endpoint: public
-          service: kafka
     image_repo_sync:
       services:
         - endpoint: internal
@@ -95,12 +65,6 @@ dependencies:
       services:
         - endpoint: internal
           service: fluentd
-    tests:
-      services:
-        - endpoint: internal
-          service: elasticsearch
-        - endpoint: internal
-          service: fluentd
 
 conf:
   fluentd:
@@ -436,40 +400,10 @@ conf:
         Time_Format %Y-%m-%dT%H:%M:%S.%L
         Time_Keep true
         Time_Key time
-
   fluentd_exporter:
     log:
       format: "logger:stdout?json=true"
       level: "info"
-  templates:
-    fluent:
-      template: "logstash-*"
-      index_patterns: "logstash-*"
-      settings:
-        number_of_shards: 1
-      mappings:
-        fluent:
-          properties:
-            kubernetes:
-              properties:
-                container_name:
-                  type: keyword
-                  index: false
-                docker_id:
-                  type: keyword
-                  index: false
-                host:
-                  type: keyword
-                  index: false
-                namespace_name:
-                  type: keyword
-                  index: false
-                pod_id:
-                  type: keyword
-                  index: false
-                pod_name:
-                  type: keyword
-                  index: false
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -506,21 +440,6 @@ endpoints:
     port:
       http:
         default: 80
-  kafka:
-    namespace: null
-    name: kafka
-    hosts:
-      default: kafka-logging
-      public: kafka
-    host_fqdn_override:
-      default: null
-    path:
-      default: null
-    scheme:
-      default: http
-    port:
-      service:
-        default: 9092
   fluentd:
     namespace: null
     name: fluentd
@@ -577,34 +496,13 @@ network_policy:
 
 pod:
   security_context:
-    daemon:
+    fluentd:
-      pod:
-        runAsUser: 65534
-      container:
-        fluentbit:
-          runAsUser: 0
-          readOnlyRootFilesystem: true
-    internal:
       pod:
         runAsUser: 65534
       container:
         fluentd:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
-    elasticsearch_template:
-      pod:
-        runAsUser: 65534
-      container:
-        elasticsearch_template:
-          allowPrivilegeEscalation: false
-          readOnlyRootFilesystem: true
-    test:
-      pod:
-        runAsUser: 65534
-      container:
-        helm_test:
-          allowPrivilegeEscalation: false
-          readOnlyRootFilesystem: true
     exporter:
       pod:
         runAsUser: 65534
@@ -622,12 +520,6 @@ pod:
         default: 10
   lifecycle:
     upgrades:
-      daemonsets:
-        pod_replacement_strategy: RollingUpdate
-        fluentbit:
-          enabled: true
-          min_ready_seconds: 0
-          max_unavailable: 1
       deployments:
         revision_history: 3
         pod_replacement_strategy: RollingUpdate
@@ -644,13 +536,6 @@ pod:
     prometheus_fluentd_exporter: 1
   resources:
     enabled: false
-    fluentbit:
-      limits:
-        memory: '400Mi'
-        cpu: '400m'
-      requests:
-        memory: '100Mi'
-        cpu: '100m'
     fluentd:
       limits:
         memory: '1024Mi'
@@ -665,47 +550,15 @@ pod:
       requests:
         memory: "128Mi"
         cpu: "500m"
-    jobs:
-      image_repo_sync:
-        requests:
-          memory: "128Mi"
-          cpu: "100m"
-        limits:
-          memory: "1024Mi"
-          cpu: "2000m"
-      tests:
-        requests:
-          memory: '128Mi'
-          cpu: '100m'
-        limits:
-          memory: '1024Mi'
-          cpu: '2000m'
-  tolerations:
-    fluentbit:
-      enabled: false
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-      - key: node-role.kubernetes.io/node
-        operator: Exists
   mounts:
     fluentd:
       fluentd:
-    fluentbit:
-      fluentbit:
-    fluent_tests:
-      fluent_tests:
-    elasticsearch_template:
-      init_container:
-      elasticsearch_template:
 
 manifests:
   configmap_bin: true
   configmap_etc: true
   deployment_fluentd: true
-  daemonset_fluentbit: true
   job_image_repo_sync: true
-  helm_tests: true
   monitoring:
     prometheus:
       configmap_bin_exporter: true
@@ -715,4 +568,3 @@ manifests:
   network_policy: false
   secret_elasticsearch: true
   service_fluentd: true
-  job_elasticsearch_template: true

tools/deployment/armada/manifests/armada-lma.yaml

@@ -328,6 +328,81 @@ data:
             master: "-Xms512m -Xmx512m"
         snapshots:
           enabled: true
+      templates:
+        syslog:
+          template: "syslog-*"
+          index_patterns: "syslog-*"
+          settings:
+            number_of_shards: 1
+          mappings:
+            syslog:
+              properties:
+                cluster:
+                  type: keyword
+                app:
+                  type: keyword
+                pid:
+                  type: integer
+                host:
+                  type: keyword
+                log:
+                  type: text
+        oslo_openstack_fluentd:
+          template: "openstack-*"
+          index_patterns: "openstack-*"
+          settings:
+            number_of_shards: 1
+          mappings:
+            oslo_openstack_fluentd:
+              properties:
+                extra:
+                  properties:
+                    project:
+                      type: text
+                      norms: false
+                    version:
+                      type: text
+                      norms: false
+                filename:
+                  type: text
+                  norms: false
+                funcname:
+                  type: text
+                  norms: false
+                message:
+                  type: text
+                  norms: false
+                process_name:
+                  type: keyword
+                  index: false
+        docker_fluentd:
+          template: "logstash-*"
+          index_patterns: "logstash-*"
+          settings:
+            number_of_shards: 1
+          mappings:
+            docker_fluentd:
+              properties:
+                kubernetes:
+                  properties:
+                    container_name:
+                      type: keyword
+                      index: false
+                    docker_id:
+                      type: keyword
+                      index: false
+                    host:
+                      type: keyword
+                      index: false
+                    namespace_name:
+                      type: keyword
+                      index: false
+                    pod_id:
+                      type: keyword
+                      index: false
+                    pod_name:
+                      type: keyword
+                      index: false
   source:
     type: local
     location: ${OSH_INFRA_PATH}
@@ -339,19 +414,17 @@ data:
 schema: armada/Chart/v1
 metadata:
   schema: metadata/Document/v1
-  name: fluent-logging
+  name: fluentbit
 data:
-  chart_name: fluent-logging
+  chart_name: fluentbit
-  release: fluent-logging
+  release: fluentbit
   namespace: osh-infra
   wait:
     timeout: 3600
     labels:
-      release_group: osh-infra-fluent-logging
+      release_group: osh-infra-fluentbit
     resources:
       - type: daemonset
-      - type: deployment
-      - type: job
   test:
     timeout: 600
   install:
@@ -362,10 +435,52 @@ data:
       delete:
         - type: job
           labels:
-            release_group: osh-infra-fluent-logging
+            release_group: osh-infra-fluentbit
         - type: pod
           labels:
-            release_group: osh-infra-fluent-logging
+            release_group: osh-infra-fluentbit
+            component: test
+  values:
+    release_uuid: ${RELEASE_UUID}
+      fluentbit:
+        node_selector_key: openstack-control-plane
+        node_selector_value: enabled
+  source:
+    type: local
+    location: ${OSH_INFRA_PATH}
+    subpath: fluentbit
+    reference: master
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: fluentd
+data:
+  chart_name: fluentd
+  release: fluentd
+  namespace: osh-infra
+  wait:
+    timeout: 3600
+    labels:
+      release_group: osh-infra-fluentd
+    resources:
+      - type: deployment
+  test:
+    timeout: 600
+  install:
+    no_hooks: False
+  upgrade:
+    no_hooks: False
+    pre:
+      delete:
+        - type: job
+          labels:
+            release_group: osh-infra-fluentd
+        - type: pod
+          labels:
+            release_group: osh-infra-fluentd
             component: test
   values:
     release_uuid: ${RELEASE_UUID}
@@ -384,9 +499,6 @@ data:
       fluentd:
         node_selector_key: openstack-control-plane
         node_selector_value: enabled
-      fluentbit:
-        node_selector_key: openstack-control-plane
-        node_selector_value: enabled
       prometheus_fluentd_exporter:
         node_selector_key: openstack-control-plane
         node_selector_value: enabled
@@ -396,7 +508,7 @@ data:
   source:
     type: local
     location: ${OSH_INFRA_PATH}
-    subpath: fluent-logging
+    subpath: fluentd
     reference: master
   dependencies:
     - helm-toolkit

tools/deployment/common/fluentbit.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Lint and package chart
+make fluentbit
+
+helm upgrade --install fluentbit ./fluentbit \
+    --namespace=osh-infra
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh osh-infra
+
+#NOTE: Validate Deployment info
+helm status fluentbit

tools/deployment/multinode/125-fluentbit.sh

@@ -0,0 +1 @@
+../common/fluentbit.sh

tools/deployment/multinode/130-fluentd.sh

@@ -17,10 +17,10 @@
 set -xe
 
 #NOTE: Lint and package chart
-make fluent-logging
+make fluentd
 
 if [ ! -d "/var/log/journal" ]; then
-tee /tmp/fluent-logging.yaml << EOF
+tee /tmp/fluentd.yaml << EOF
 monitoring:
   prometheus:
     enabled: true
@@ -36,20 +36,16 @@ pod:
           - name: runlog
             mountPath: /run/log
 EOF
-helm upgrade --install fluent-logging ./fluent-logging \
+helm upgrade --install fluentd ./fluentd \
     --namespace=osh-infra \
-    --values=/tmp/fluent-logging.yaml
+    --values=/tmp/fluentd.yaml
 else
-helm upgrade --install fluent-logging ./fluent-logging \
+helm upgrade --install fluentd ./fluentd \
-    --namespace=osh-infra \
+    --namespace=osh-infra
-    --set monitoring.prometheus.enabled=true
 fi
 
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh osh-infra
 
 #NOTE: Validate Deployment info
-helm status fluent-logging
+helm status fluentd
-
-#NOTE: Run helm tests
-helm test fluent-logging

tools/deployment/osh-infra-logging/060-fluentbit.sh

@@ -0,0 +1 @@
+../common/fluentbit.sh

tools/deployment/osh-infra-logging/065-fluentd.sh

@@ -17,10 +17,10 @@
 set -xe
 
 #NOTE: Lint and package chart
-make fluent-logging
+make fluentd
 
 if [ ! -d "/var/log/journal" ]; then
-tee /tmp/fluent-logging.yaml << EOF
+tee /tmp/fluentd.yaml << EOF
 monitoring:
   prometheus:
     enabled: true
@@ -38,11 +38,11 @@ pod:
           - name: runlog
             mountPath: /run/log
 EOF
-helm upgrade --install fluent-logging ./fluent-logging \
+helm upgrade --install fluentd ./fluentd \
     --namespace=osh-infra \
-    --values=/tmp/fluent-logging.yaml
+    --values=/tmp/fluentd.yaml
 else
-helm upgrade --install fluent-logging ./fluent-logging \
+helm upgrade --install fluentd ./fluentd \
     --namespace=osh-infra \
     --set pod.replicas.fluentd=1 \
     --set monitoring.prometheus.enabled=true
@@ -52,6 +52,4 @@ fi
 ./tools/deployment/common/wait-for-pods.sh osh-infra
 
 #NOTE: Validate Deployment info
-helm status fluent-logging
+helm status fluentd
-
-helm test fluent-logging

zuul.d/jobs.yaml

@@ -60,7 +60,8 @@
         - ./tools/deployment/multinode/110-nagios.sh
         - ./tools/deployment/multinode/115-radosgw-osh-infra.sh
         - ./tools/deployment/multinode/120-elasticsearch.sh
-        - ./tools/deployment/multinode/130-fluent-logging.sh
+        - ./tools/deployment/multinode/125-fluentbit.sh
+        - ./tools/deployment/multinode/130-fluentd.sh
         - ./tools/deployment/multinode/140-kibana.sh
         - ./tools/deployment/multinode/600-grafana-selenium.sh
         - ./tools/deployment/multinode/610-nagios-selenium.sh
@@ -127,7 +128,8 @@
         - ./tools/deployment/osh-infra-logging/040-ldap.sh
         - ./tools/deployment/osh-infra-logging/050-elasticsearch.sh
         - ./tools/deployment/osh-infra-logging/055-elasticsearch-ldap.sh
-        - ./tools/deployment/osh-infra-logging/060-fluent-logging.sh
+        - ./tools/deployment/osh-infra-logging/060-fluentbit.sh
+        - ./tools/deployment/osh-infra-logging/065-fluentd.sh
         - ./tools/deployment/osh-infra-logging/070-kibana.sh
         - ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh