Merge "Separate fluentbit and fluentd charts"

doc/source/install/multinode.rst

@@ -210,10 +210,10 @@ Alternatively, this step can be performed by running the script directly:
 
   ./tools/deployment/multinode/120-elasticsearch.sh
 
-Deploy Fluent-Logging
+Deploy Fluentbit
-^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^
 
-.. literalinclude:: ../../../tools/deployment/multinode/130-fluent-logging.sh
+.. literalinclude:: ../../../tools/deployment/multinode/125-fluentbit.sh
     :language: shell
     :lines: 1,17-
 
@@ -221,4 +221,17 @@ Alternatively, this step can be performed by running the script directly:
 
 .. code-block:: shell
 
-  ./tools/deployment/multinode/130-fluent-logging.sh
+  ./tools/deployment/multinode/125-fluentbit.sh
+
+Deploy Fluentd
+^^^^^^^^^^^^^^
+
+.. literalinclude:: ../../../tools/deployment/multinode/130-fluentd.sh
+    :language: shell
+    :lines: 1,17-
+
+Alternatively, this step can be performed by running the script directly:
+
+.. code-block:: shell
+
+  ./tools/deployment/multinode/130-fluentd.sh

elasticsearch/templates/bin/_helm-tests.sh.tpl

@@ -52,6 +52,24 @@ function check_snapshot_repositories () {
   {{ end }}
 }
 
+{{ if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }}
+# Tests whether elasticsearch has successfully generated the elasticsearch index mapping
+# templates defined by values.yaml
+function check_templates () {
+  {{ range $template, $fields := .Values.conf.templates }}
+  {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+              -XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \
+              | python -c "import sys, json; print len(json.load(sys.stdin))")
+  if [ "${{$template}}_total_hits" -gt 0 ]; then
+     echo "PASS: Successful hits on {{$template}} template!"
+  else
+     echo "FAIL: No hits on query for {{$template}} template! Exiting";
+     exit 1;
+  fi
+  {{ end }}
+}
+{{ end }}
+
 function remove_test_index () {
   echo "Deleting index created for service testing"
   curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
@@ -63,4 +81,5 @@ create_test_index
 {{ if .Values.conf.elasticsearch.snapshots.enabled }}
 check_snapshot_repositories
 {{ end }}
+check_templates
 remove_test_index

elasticsearch/templates/configmap-bin-elasticsearch.yaml

@@ -38,6 +38,8 @@ data:
 {{ tuple "bin/_register-repository.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   es-cluster-wait.sh: |
 {{ tuple "bin/_es-cluster-wait.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  create_template.sh: |
+{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   image-repo-sync.sh: |
 {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
 {{- end }}

elasticsearch/templates/configmap-etc-templates.yaml

@@ -1,11 +1,11 @@
 {{/*
-Copyright 2017-2018 The Openstack-Helm Authors.
+Copyright 2019 The Openstack-Helm Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+   http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +14,17 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and .Values.manifests.monitoring.prometheus.network_policy_exporter .Values.monitoring.prometheus.enabled -}}
+{{- if .Values.manifests.configmap_etc_templates }}
-{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-fluentd-exporter" }}
+{{- $envAll := . }}
-{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
+
-{{- end -}}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: elasticsearch-templates-etc
+type: Opaque
+data:
+{{ range $template, $fields := .Values.conf.templates }}
+  {{ $template }}.json: {{ toJson $fields | b64enc }}
+{{ end }}
+{{- end }}

elasticsearch/templates/job-elasticsearch-template.yaml

@@ -14,26 +14,26 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }}
+{{- if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }}
 {{- $envAll := . }}
 {{- $esUserSecret := .Values.secrets.elasticsearch.user }}
-{{- $mounts_elasticsearch_template := .Values.pod.mounts.elasticsearch_template.elasticsearch_template }}
+{{- $mounts_elasticsearch_templates := .Values.pod.mounts.elasticsearch_templates.elasticsearch_templates }}
-{{- $mounts_elasticsearch_template_init := .Values.pod.mounts.elasticsearch_template.init_container }}
+{{- $mounts_elasticsearch_templates_init := .Values.pod.mounts.elasticsearch_templates.init_container }}
 
-{{- $serviceAccountName := "fluent-logging-elasticsearch-template" }}
+{{- $serviceAccountName := "create-elasticsearch-templates" }}
-{{ tuple $envAll "elasticsearch_template" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+{{ tuple $envAll "elasticsearch_templates" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: elasticsearch-template
+  name: create-elasticsearch-templates
   annotations:
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
 spec:
   template:
     metadata:
       labels:
-{{ tuple $envAll "fluentd" "elasticsearch-template" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+{{ tuple $envAll "elasticsearch" "create-templates" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "elasticsearch_template" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}
@@ -41,11 +41,11 @@ spec:
       nodeSelector:
         {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }}
       initContainers:
-{{ tuple $envAll "elasticsearch_template" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+{{ tuple $envAll "elasticsearch_templates" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
-        - name: elasticsearch-template
+        - name: create-elasticsearch-templates
-{{ tuple $envAll "elasticsearch_template" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll "elasticsearch_templates" | include "helm-toolkit.snippets.image" | indent 10 }}
-{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_template | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_templates | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "elasticsearch_template" "container" "elasticsearch_template" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           env:
             - name: ELASTICSEARCH_HOST
@@ -67,27 +67,27 @@ spec:
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
-            - name: fluent-logging-bin
+            - name: elasticsearch-bin
               mountPath: /tmp/create_template.sh
               subPath: create_template.sh
               readOnly: true
             {{ range $template, $fields := .Values.conf.templates }}
-            - name: fluent-logging-etc
+            - name: elasticsearch-templates-etc
               mountPath: /tmp/{{$template}}.json
               subPath: {{$template}}.json
               readOnly: true
             {{ end }}
-{{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 12 }}{{ end }}
+{{ if $mounts_elasticsearch_templates.volumeMounts }}{{ toYaml $mounts_elasticsearch_templates.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: pod-tmp
           emptyDir: {}
-        - name: fluent-logging-bin
+        - name: elasticsearch-bin
           configMap:
-            name: fluent-logging-bin
+            name: elasticsearch-bin
             defaultMode: 0555
-        - name: fluent-logging-etc
+        - name: elasticsearch-templates-etc
           secret:
-            secretName: fluent-logging-etc
+            secretName: elasticsearch-templates-etc
             defaultMode: 0444
-{{ if $mounts_elasticsearch_template.volumes }}{{ toYaml $mounts_elasticsearch_template.volumes | indent 8 }}{{ end }}
+{{ if $mounts_elasticsearch_templates.volumes }}{{ toYaml $mounts_elasticsearch_templates.volumes | indent 8 }}{{ end }}
 {{- end }}

elasticsearch/values.yaml

@@ -30,6 +30,7 @@ images:
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
     snapshot_repository: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_xenial
     es_cluster_wait: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_xenial
+    elasticsearch_templates: docker.io/openstackhelm/heat:newton
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: "IfNotPresent"
   local_registry:
@@ -82,6 +83,10 @@ dependencies:
       services:
         - endpoint: internal
           service: elasticsearch
+    elasticsearch_templates:
+      services: null
+      jobs:
+        - elasticsearch-cluster-wait
     image_repo_sync:
       services:
         - endpoint: internal
@@ -220,6 +225,8 @@ pod:
   mounts:
     elasticsearch:
       elasticsearch:
+    elasticsearch_templates:
+      elasticsearch_templates:
   resources:
     enabled: false
     apache_proxy:
@@ -265,6 +272,13 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+      elasticsearch_templates:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
       image_repo_sync:
         requests:
           memory: "128Mi"
@@ -635,6 +649,35 @@ conf:
     es:
       all: true
       timeout: 20s
+  templates:
+    fluent:
+      template: "logstash-*"
+      index_patterns: "logstash-*"
+      settings:
+        number_of_shards: 1
+      mappings:
+        fluent:
+          properties:
+            kubernetes:
+              properties:
+                container_name:
+                  type: keyword
+                  index: false
+                docker_id:
+                  type: keyword
+                  index: false
+                host:
+                  type: keyword
+                  index: false
+                namespace_name:
+                  type: keyword
+                  index: false
+                pod_id:
+                  type: keyword
+                  index: false
+                pod_name:
+                  type: keyword
+                  index: false
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -771,11 +814,13 @@ manifests:
   configmap_bin_elasticsearch: true
   configmap_etc_curator: true
   configmap_etc_elasticsearch: true
+  configmap_etc_templates: true
   cron_curator: true
   deployment_client: true
   deployment_master: true
   ingress: true
   job_cluster_wait: true
+  job_elasticsearch_templates: true
   job_image_repo_sync: true
   job_snapshot_repository: true
   job_s3_user: true

fluent-logging/README.rst

@@ -1,30 +0,0 @@
-Fluentd-logging
-===============
-
-OpenStack-Helm defines a centralized logging mechanism to provide insight into
-the state of the OpenStack services and infrastructure components as
-well as underlying kubernetes platform. Among the requirements for a logging
-platform, where log data can come from and where log data need to be delivered
-are very variable. To support various logging scenarios, OpenStack-Helm should
-provide a flexible mechanism to meet with certain operation needs. This chart
-proposes fast and lightweight log forwarder and full featured log aggregator
-complementing each other providing a flexible and reliable solution. Especially,
-Fluent-bit is proposed as a log forwarder and Fluentd is proposed as a main log
-aggregator and processor.
-
-
-Mechanism
----------
-
-Fluent-bit, Fluentd meet OpenStack-Helm's logging requirements for gathering,
-aggregating, and delivering of logged events. Flunt-bit runs as a daemonset on
-each node and mounts the /var/lib/docker/containers directory. The Docker
-container runtime engine directs events posted to stdout and stderr to this
-directory on the host. Fluent-bit then forward the contents of that directory to
-Fluentd. Fluentd runs as deployment at the designated nodes and expose service
-for Fluent-bit to foward logs. Fluentd should then apply the Logstash format to
-the logs. Fluentd can also write kubernetes and OpenStack metadata to the logs.
-Fluentd will then forward the results to Elasticsearch and to optionally kafka.
-Elasticsearch indexes the logs in a logstash-* index by default. kafka stores
-the logs in a 'logs' topic by default. Any external tool can then consume the
-'logs' topic.

fluent-logging/templates/bin/_helm-tests.sh.tpl

@@ -1,41 +0,0 @@
-#!/bin/bash
-
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-set -ex
-
-{{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }}
-# Tests whether fluent-logging has successfully generated the elasticsearch index mapping
-# templates defined by values.yaml
-function check_templates () {
-  {{ range $template, $fields := .Values.conf.templates }}
-  {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-              -XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \
-              | python -c "import sys, json; print len(json.load(sys.stdin))")
-  if [ "${{$template}}_total_hits" -gt 0 ]; then
-     echo "PASS: Successful hits on {{$template}} template, provided by fluent-logging!"
-  else
-     echo "FAIL: No hits on query for {{$template}} template! Exiting";
-     exit 1;
-  fi
-  {{ end }}
-}
-{{ end }}
-
-{{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }}
-check_templates
-{{ end }}

fluent-logging/templates/pod-helm-tests.yaml

@@ -1,75 +0,0 @@
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- if .Values.manifests.helm_tests }}
-{{- $envAll := . }}
-{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
-
-{{- $serviceAccountName := print .Release.Name "-test" }}
-{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: "{{.Release.Name}}-test"
-  labels:
-{{ tuple $envAll "fluentd" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
-  annotations:
-    "helm.sh/hook": test-success
-    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
-spec:
-{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }}
-  serviceAccountName: {{ $serviceAccountName }}
-  nodeSelector:
-    {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }}
-  restartPolicy: Never
-  initContainers:
-{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }}
-  containers:
-    - name: {{.Release.Name}}-helm-tests
-{{ tuple $envAll "helm_tests" | include "helm-toolkit.snippets.image" | indent 6 }}
-{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }}
-{{ dict "envAll" $envAll "application" "test" "container" "helm_tests" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }}
-      command:
-        - /tmp/helm-tests.sh
-      env:
-        - name: ELASTICSEARCH_USERNAME
-          valueFrom:
-            secretKeyRef:
-              name: {{ $esUserSecret }}
-              key: ELASTICSEARCH_USERNAME
-        - name: ELASTICSEARCH_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: {{ $esUserSecret }}
-              key: ELASTICSEARCH_PASSWORD
-        - name: ELASTICSEARCH_ENDPOINT
-          value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
-      volumeMounts:
-        - name: pod-tmp
-          mountPath: /tmp
-        - name: fluent-logging-bin
-          mountPath: /tmp/helm-tests.sh
-          subPath: helm-tests.sh
-          readOnly: true
-  volumes:
-    - name: pod-tmp
-      emptyDir: {}
-    - name: fluent-logging-bin
-      configMap:
-        name: fluent-logging-bin
-        defaultMode: 0555
-{{- end }}

fluentbit/Chart.yaml

@@ -0,0 +1,24 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+description: OpenStack-Helm Fluentbit
+name: fluentbit
+version: 0.1.0
+home: http://www.fluentbit.io/
+sources:
+  - https://github.com/fluent/fluentbit
+  - https://git.openstack.org/cgit/openstack/openstack-helm-infra
+maintainers:
+  - name: OpenStack-Helm Authors

fluentbit/templates/configmap-bin.yaml

@@ -20,16 +20,10 @@ limitations under the License.
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: fluent-logging-bin
+  name: fluentbit-bin
 data:
-  fluentd.sh: |
-{{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   fluent-bit.sh: |
 {{ tuple "bin/_fluent-bit.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  helm-tests.sh: |
-{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  create_template.sh: |
-{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   image-repo-sync.sh: |
 {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
 {{- end }}

fluentbit/templates/configmap-etc.yaml

@@ -20,13 +20,9 @@ limitations under the License.
 apiVersion: v1
 kind: Secret
 metadata:
-  name: fluent-logging-etc
+  name: fluentbit-etc
 type: Opaque
 data:
   fluent-bit.conf: {{ .Values.conf.fluentbit.template | b64enc }}
   parsers.conf: {{ .Values.conf.parsers.template | b64enc }}
-  fluent.conf: {{ .Values.conf.fluentd.template | b64enc }}
-{{ range $template, $fields := .Values.conf.templates }}
-  {{ $template }}.json: {{ toJson $fields | b64enc }}
-{{ end }}
 {{- end }}

fluentbit/templates/daemonset-fluent-bit.yaml

@@ -95,7 +95,7 @@ spec:
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
 {{ dict "envAll" $envAll "podName" "fluentbit" "containerNames" (list "fluentbit") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
-{{ dict "envAll" $envAll "application" "daemon" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+{{ dict "envAll" $envAll "application" "fluentbit" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}
 {{ if $envAll.Values.pod.tolerations.fluentbit.enabled }}
 {{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
@@ -112,7 +112,7 @@ spec:
         - name: fluentbit
 {{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-{{ dict "envAll" $envAll "application" "daemon" "container" "fluentbit" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ dict "envAll" $envAll "application" "fluentbit" "container" "fluentbit" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           command:
             - /tmp/fluent-bit.sh
           env:
@@ -123,7 +123,7 @@ spec:
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
-            - name: fluent-logging-bin
+            - name: fluentbit-bin
               mountPath: /tmp/fluent-bit.sh
               subPath: fluent-bit.sh
               readOnly: true
@@ -133,11 +133,11 @@ spec:
             - name: varlibdockercontainers
               mountPath: /var/lib/docker/containers
               readOnly: true
-            - name: fluent-logging-etc
+            - name: fluentbit-etc
               mountPath: /fluent-bit/etc/fluent-bit.conf
               subPath: fluent-bit.conf
               readOnly: true
-            - name: fluent-logging-etc
+            - name: fluentbit-etc
               mountPath: /fluent-bit/etc/parsers.conf
               subPath: parsers.conf
               readOnly: true
@@ -151,13 +151,13 @@ spec:
         - name: varlibdockercontainers
           hostPath:
             path: /var/lib/docker/containers
-        - name: fluent-logging-bin
+        - name: fluentbit-bin
           configMap:
-            name: fluent-logging-bin
+            name: fluentbit-bin
             defaultMode: 0555
-        - name: fluent-logging-etc
+        - name: fluentbit-etc
           secret:
-            secretName: fluent-logging-etc
+            secretName: fluentbit-etc
             defaultMode: 0444
 {{ if $mounts_fluentbit.volumes }}{{ toYaml $mounts_fluentbit.volumes | indent 8 }}{{ end }}
 {{- end }}

fluentbit/templates/job-image-repo-sync.yaml

@@ -15,6 +15,6 @@ limitations under the License.
 */}}
 
 {{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }}
-{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluent-logging" -}}
+{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluentbit" -}}
 {{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }}
 {{- end }}

fluentbit/values.yaml

@@ -0,0 +1,256 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default values for fluentbit
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+release_group: null
+
+labels:
+  fluentbit:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+
+images:
+  tags:
+    fluentbit: docker.io/fluent/fluent-bit:0.14.2
+    dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
+    image_repo_sync: docker.io/docker:17.07.0
+  pull_policy: IfNotPresent
+  local_registry:
+    active: false
+    exclude:
+      - dep_check
+      - image_repo_sync
+
+dependencies:
+  dynamic:
+    common:
+      local_image_registry:
+        jobs:
+          - fluentbit-image-repo-sync
+        services:
+          - endpoint: node
+            service: local_image_registry
+  static:
+    image_repo_sync:
+      services:
+        - endpoint: internal
+          service: local_image_registry
+
+conf:
+  fluentbit:
+    template: |
+      [SERVICE]
+          Daemon false
+          Flush 30
+          Log_Level info
+          Parsers_File parsers.conf
+
+      [INPUT]
+          Buffer_Chunk_Size 1M
+          Buffer_Max_Size 1M
+          Mem_Buf_Limit 5MB
+          Name tail
+          Path /var/log/kern.log
+          Tag kernel
+
+      [INPUT]
+          Buffer_Chunk_Size 1M
+          Buffer_Max_Size 1M
+          Mem_Buf_Limit 5MB
+          Name tail
+          Parser docker
+          Path /var/log/containers/*.log
+          Tag kube.*
+
+      [INPUT]
+          Buffer_Chunk_Size 1M
+          Buffer_Max_Size 1M
+          Mem_Buf_Limit 5MB
+          Name tail
+          Path /var/log/libvirt/libvirtd.log
+          Tag libvirt
+
+      [INPUT]
+          Buffer_Chunk_Size 1M
+          Buffer_Max_Size 1M
+          Mem_Buf_Limit 5MB
+          Name tail
+          Path /var/log/libvirt/qemu/*.log
+          Tag qemu
+
+      [INPUT]
+          Buffer_Chunk_Size 1M
+          Buffer_Max_Size 1M
+          Mem_Buf_Limit 5MB
+          Name systemd
+          Path ${JOURNAL_PATH}
+          Systemd_Filter _SYSTEMD_UNIT=kubelet.service
+          Tag journal.*
+
+      [INPUT]
+          Buffer_Chunk_Size 1M
+          Buffer_Max_Size 1M
+          Mem_Buf_Limit 5MB
+          Name systemd
+          Path ${JOURNAL_PATH}
+          Systemd_Filter _SYSTEMD_UNIT=docker.service
+          Tag journal.*
+
+      [FILTER]
+          Interval 1s
+          Match **
+          Name throttle
+          Rate 1000
+          Window 300
+
+      [FILTER]
+          Match libvirt
+          Name record_modifier
+          Record hostname ${HOSTNAME}
+
+      [FILTER]
+          Match qemu
+          Name record_modifier
+          Record hostname ${HOSTNAME}
+
+      [FILTER]
+          Match kernel
+          Name record_modifier
+          Record hostname ${HOSTNAME}
+
+      [FILTER]
+          Match journal.**
+          Name modify
+          Rename _BOOT_ID BOOT_ID
+          Rename _CAP_EFFECTIVE CAP_EFFECTIVE
+          Rename _CMDLINE CMDLINE
+          Rename _COMM COMM
+          Rename _EXE EXE
+          Rename _GID GID
+          Rename _HOSTNAME HOSTNAME
+          Rename _MACHINE_ID MACHINE_ID
+          Rename _PID PID
+          Rename _SYSTEMD_CGROUP SYSTEMD_CGROUP
+          Rename _SYSTEMD_SLICE SYSTEMD_SLICE
+          Rename _SYSTEMD_UNIT SYSTEMD_UNIT
+          Rename _TRANSPORT TRANSPORT
+          Rename _UID UID
+
+      [OUTPUT]
+          Match **.fluentd**
+          Name null
+
+      [FILTER]
+          Match kube.*
+          Merge_JSON_Log true
+          Name kubernetes
+
+      [OUTPUT]
+          Host ${FLUENTD_HOST}
+          Match *
+          Name forward
+          Port ${FLUENTD_PORT}
+  parsers:
+    template: |
+      [PARSER]
+        Decode_Field_As escaped_utf8 log
+        Format json
+        Name docker
+        Time_Format %Y-%m-%dT%H:%M:%S.%L
+        Time_Keep true
+        Time_Key time
+
+endpoints:
+  cluster_domain_suffix: cluster.local
+  local_image_registry:
+    name: docker-registry
+    namespace: docker-registry
+    hosts:
+      default: localhost
+      internal: docker-registry
+      node: localhost
+    host_fqdn_override:
+      default: null
+    port:
+      registry:
+        node: 5000
+  fluentd:
+    namespace: null
+    name: fluentd
+    hosts:
+      default: fluentd-logging
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      service:
+        default: 24224
+      metrics:
+        default: 24220
+
+pod:
+  security_context:
+    fluentbit:
+      pod:
+        runAsUser: 65534
+      container:
+        fluentbit:
+          runAsUser: 0
+          readOnlyRootFilesystem: false
+  affinity:
+    anti:
+      type:
+        default: preferredDuringSchedulingIgnoredDuringExecution
+      topologyKey:
+        default: kubernetes.io/hostname
+  lifecycle:
+    upgrades:
+      daemonsets:
+        pod_replacement_strategy: RollingUpdate
+        fluentbit:
+          enabled: true
+          min_ready_seconds: 0
+          max_unavailable: 1
+  resources:
+    enabled: false
+    fluentbit:
+      limits:
+        memory: '400Mi'
+        cpu: '400m'
+      requests:
+        memory: '100Mi'
+        cpu: '100m'
+  tolerations:
+    fluentbit:
+      enabled: false
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      - key: node-role.kubernetes.io/node
+        operator: Exists
+  mounts:
+    fluentbit:
+      fluentbit:
+
+manifests:
+  configmap_bin: true
+  configmap_etc: true
+  daemonset_fluentbit: true
+  job_image_repo_sync: true

fluentd/Chart.yaml

@@ -14,11 +14,10 @@
 
 apiVersion: v1
 description: OpenStack-Helm Fluentd
-name: fluent-logging
+name: fluentd
 version: 0.1.0
 home: http://www.fluentbit.io/
 sources:
-  - https://github.com/fluent/fluentbit
   - https://github.com/fluent/fluentd
   - https://git.openstack.org/cgit/openstack/openstack-helm-infra
 maintainers:

fluentd/requirements.yaml

@@ -0,0 +1,18 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dependencies:
+  - name: helm-toolkit
+    repository: http://localhost:8879/charts/
+    version: 0.1.0

fluentd/templates/configmap-bin.yaml

@@ -0,0 +1,29 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_bin }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fluentd-bin
+data:
+  fluentd.sh: |
+{{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  image-repo-sync.sh: |
+{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
+{{- end }}

fluentd/templates/configmap-etc.yaml

@@ -0,0 +1,27 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_etc }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: fluentd-etc
+type: Opaque
+data:
+  fluent.conf: {{ .Values.conf.fluentd.template | b64enc }}
+{{- end }}

fluentd/templates/deployment-fluentd.yaml

@@ -95,7 +95,7 @@ spec:
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
     spec:
-{{ dict "envAll" $envAll "application" "internal" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+{{ dict "envAll" $envAll "application" "fluentd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}
       affinity:
 {{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
@@ -108,7 +108,7 @@ spec:
         - name: fluentd
 {{ tuple $envAll "fluentd" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.fluentd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-{{ dict "envAll" $envAll "application" "internal" "container" "fluentd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ dict "envAll" $envAll "application" "fluentd" "container" "fluentd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           command:
             - /tmp/fluentd.sh
             - start
@@ -149,11 +149,11 @@ spec:
               mountPath: /tmp
             - name: pod-etc-fluentd
               mountPath: /fluentd/etc
-            - name: fluent-logging-etc
+            - name: fluentd-etc
               mountPath: /fluentd/etc/fluent.conf
               subPath: fluent.conf
               readOnly: true
-            - name: fluent-logging-bin
+            - name: fluentd-bin
               mountPath: /tmp/fluentd.sh
               subPath: fluentd.sh
               readOnly: true
@@ -163,13 +163,13 @@ spec:
           emptyDir: {}
         - name: pod-etc-fluentd
           emptyDir: {}
-        - name: fluent-logging-etc
+        - name: fluentd-etc
           secret:
-            secretName: fluent-logging-etc
+            secretName: fluentd-etc
             defaultMode: 0444
-        - name: fluent-logging-bin
+        - name: fluentd-bin
           configMap:
-            name: fluent-logging-bin
+            name: fluentd-bin
             defaultMode: 0555
 {{- if $mounts_fluentd.volumes }}{{ toYaml $mounts_fluentd.volumes | indent 8 }}{{- end }}
 {{- end }}

fluentd/templates/job-image-repo-sync.yaml

@@ -0,0 +1,20 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }}
+{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluentd" -}}
+{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }}
+{{- end }}

fluentd/values.yaml

@@ -22,22 +22,12 @@ labels:
   fluentd:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
-  fluentbit:
-    node_selector_key: openstack-control-plane
-    node_selector_value: enabled
   prometheus_fluentd_exporter:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
-  job:
-    node_selector_key: openstack-control-plane
-    node_selector_value: enabled
-  test:
-    node_selector_key: openstack-control-plane
-    node_selector_value: enabled
 
 images:
   tags:
-    fluentbit: docker.io/fluent/fluent-bit:0.14.2
     fluentd: docker.io/fluent/fluentd-kubernetes-daemonset:v1.2-debian-elasticsearch
     prometheus_fluentd_exporter: docker.io/bitnami/fluentd-exporter:0.2.0
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
@@ -60,33 +50,13 @@ dependencies:
     common:
       local_image_registry:
         jobs:
-          - fluent-logging-image-repo-sync
+          - fluentd-image-repo-sync
         services:
           - endpoint: node
             service: local_image_registry
   static:
-    elasticsearch_template:
-      services:
-        - endpoint: internal
-          service: elasticsearch
-    fluentbit:
-      jobs:
-        - elasticsearch-template
-      services:
-        - endpoint: internal
-          service: fluentd
     fluentd:
-      jobs:
+      services: null
-        - elasticsearch-template
-      services:
-        - endpoint: internal
-          service: elasticsearch
-    fluentd_with_kafka:
-      services:
-        - endpoint: internal
-          service: elasticsearch
-        - endpoint: public
-          service: kafka
     image_repo_sync:
       services:
         - endpoint: internal
@@ -95,12 +65,6 @@ dependencies:
       services:
         - endpoint: internal
           service: fluentd
-    tests:
-      services:
-        - endpoint: internal
-          service: elasticsearch
-        - endpoint: internal
-          service: fluentd
 
 conf:
   fluentd:
@@ -436,40 +400,10 @@ conf:
         Time_Format %Y-%m-%dT%H:%M:%S.%L
         Time_Keep true
         Time_Key time
-
   fluentd_exporter:
     log:
       format: "logger:stdout?json=true"
       level: "info"
-  templates:
-    fluent:
-      template: "logstash-*"
-      index_patterns: "logstash-*"
-      settings:
-        number_of_shards: 1
-      mappings:
-        fluent:
-          properties:
-            kubernetes:
-              properties:
-                container_name:
-                  type: keyword
-                  index: false
-                docker_id:
-                  type: keyword
-                  index: false
-                host:
-                  type: keyword
-                  index: false
-                namespace_name:
-                  type: keyword
-                  index: false
-                pod_id:
-                  type: keyword
-                  index: false
-                pod_name:
-                  type: keyword
-                  index: false
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -506,21 +440,6 @@ endpoints:
     port:
       http:
         default: 80
-  kafka:
-    namespace: null
-    name: kafka
-    hosts:
-      default: kafka-logging
-      public: kafka
-    host_fqdn_override:
-      default: null
-    path:
-      default: null
-    scheme:
-      default: http
-    port:
-      service:
-        default: 9092
   fluentd:
     namespace: null
     name: fluentd
@@ -577,34 +496,13 @@ network_policy:
 
 pod:
   security_context:
-    daemon:
+    fluentd:
-      pod:
-        runAsUser: 65534
-      container:
-        fluentbit:
-          runAsUser: 0
-          readOnlyRootFilesystem: true
-    internal:
       pod:
         runAsUser: 65534
       container:
         fluentd:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
-    elasticsearch_template:
-      pod:
-        runAsUser: 65534
-      container:
-        elasticsearch_template:
-          allowPrivilegeEscalation: false
-          readOnlyRootFilesystem: true
-    test:
-      pod:
-        runAsUser: 65534
-      container:
-        helm_test:
-          allowPrivilegeEscalation: false
-          readOnlyRootFilesystem: true
     exporter:
       pod:
         runAsUser: 65534
@@ -622,12 +520,6 @@ pod:
         default: 10
   lifecycle:
     upgrades:
-      daemonsets:
-        pod_replacement_strategy: RollingUpdate
-        fluentbit:
-          enabled: true
-          min_ready_seconds: 0
-          max_unavailable: 1
       deployments:
         revision_history: 3
         pod_replacement_strategy: RollingUpdate
@@ -644,13 +536,6 @@ pod:
     prometheus_fluentd_exporter: 1
   resources:
     enabled: false
-    fluentbit:
-      limits:
-        memory: '400Mi'
-        cpu: '400m'
-      requests:
-        memory: '100Mi'
-        cpu: '100m'
     fluentd:
       limits:
         memory: '1024Mi'
@@ -665,47 +550,15 @@ pod:
       requests:
         memory: "128Mi"
         cpu: "500m"
-    jobs:
-      image_repo_sync:
-        requests:
-          memory: "128Mi"
-          cpu: "100m"
-        limits:
-          memory: "1024Mi"
-          cpu: "2000m"
-      tests:
-        requests:
-          memory: '128Mi'
-          cpu: '100m'
-        limits:
-          memory: '1024Mi'
-          cpu: '2000m'
-  tolerations:
-    fluentbit:
-      enabled: false
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-      - key: node-role.kubernetes.io/node
-        operator: Exists
   mounts:
     fluentd:
       fluentd:
-    fluentbit:
-      fluentbit:
-    fluent_tests:
-      fluent_tests:
-    elasticsearch_template:
-      init_container:
-      elasticsearch_template:
 
 manifests:
   configmap_bin: true
   configmap_etc: true
   deployment_fluentd: true
-  daemonset_fluentbit: true
   job_image_repo_sync: true
-  helm_tests: true
   monitoring:
     prometheus:
       configmap_bin_exporter: true
@@ -715,4 +568,3 @@ manifests:
   network_policy: false
   secret_elasticsearch: true
   service_fluentd: true
-  job_elasticsearch_template: true

tools/deployment/armada/manifests/armada-lma.yaml

@@ -328,6 +328,81 @@ data:
             master: "-Xms512m -Xmx512m"
         snapshots:
           enabled: true
+      templates:
+        syslog:
+          template: "syslog-*"
+          index_patterns: "syslog-*"
+          settings:
+            number_of_shards: 1
+          mappings:
+            syslog:
+              properties:
+                cluster:
+                  type: keyword
+                app:
+                  type: keyword
+                pid:
+                  type: integer
+                host:
+                  type: keyword
+                log:
+                  type: text
+        oslo_openstack_fluentd:
+          template: "openstack-*"
+          index_patterns: "openstack-*"
+          settings:
+            number_of_shards: 1
+          mappings:
+            oslo_openstack_fluentd:
+              properties:
+                extra:
+                  properties:
+                    project:
+                      type: text
+                      norms: false
+                    version:
+                      type: text
+                      norms: false
+                filename:
+                  type: text
+                  norms: false
+                funcname:
+                  type: text
+                  norms: false
+                message:
+                  type: text
+                  norms: false
+                process_name:
+                  type: keyword
+                  index: false
+        docker_fluentd:
+          template: "logstash-*"
+          index_patterns: "logstash-*"
+          settings:
+            number_of_shards: 1
+          mappings:
+            docker_fluentd:
+              properties:
+                kubernetes:
+                  properties:
+                    container_name:
+                      type: keyword
+                      index: false
+                    docker_id:
+                      type: keyword
+                      index: false
+                    host:
+                      type: keyword
+                      index: false
+                    namespace_name:
+                      type: keyword
+                      index: false
+                    pod_id:
+                      type: keyword
+                      index: false
+                    pod_name:
+                      type: keyword
+                      index: false
   source:
     type: local
     location: ${OSH_INFRA_PATH}
@@ -339,19 +414,17 @@ data:
 schema: armada/Chart/v1
 metadata:
   schema: metadata/Document/v1
-  name: fluent-logging
+  name: fluentbit
 data:
-  chart_name: fluent-logging
+  chart_name: fluentbit
-  release: fluent-logging
+  release: fluentbit
   namespace: osh-infra
   wait:
     timeout: 3600
     labels:
-      release_group: osh-infra-fluent-logging
+      release_group: osh-infra-fluentbit
     resources:
       - type: daemonset
-      - type: deployment
-      - type: job
   test:
     timeout: 600
   install:
@@ -362,10 +435,52 @@ data:
       delete:
         - type: job
           labels:
-            release_group: osh-infra-fluent-logging
+            release_group: osh-infra-fluentbit
         - type: pod
           labels:
-            release_group: osh-infra-fluent-logging
+            release_group: osh-infra-fluentbit
+            component: test
+  values:
+    release_uuid: ${RELEASE_UUID}
+      fluentbit:
+        node_selector_key: openstack-control-plane
+        node_selector_value: enabled
+  source:
+    type: local
+    location: ${OSH_INFRA_PATH}
+    subpath: fluentbit
+    reference: master
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: fluentd
+data:
+  chart_name: fluentd
+  release: fluentd
+  namespace: osh-infra
+  wait:
+    timeout: 3600
+    labels:
+      release_group: osh-infra-fluentd
+    resources:
+      - type: deployment
+  test:
+    timeout: 600
+  install:
+    no_hooks: False
+  upgrade:
+    no_hooks: False
+    pre:
+      delete:
+        - type: job
+          labels:
+            release_group: osh-infra-fluentd
+        - type: pod
+          labels:
+            release_group: osh-infra-fluentd
             component: test
   values:
     release_uuid: ${RELEASE_UUID}
@@ -384,9 +499,6 @@ data:
       fluentd:
         node_selector_key: openstack-control-plane
         node_selector_value: enabled
-      fluentbit:
-        node_selector_key: openstack-control-plane
-        node_selector_value: enabled
       prometheus_fluentd_exporter:
         node_selector_key: openstack-control-plane
         node_selector_value: enabled
@@ -396,7 +508,7 @@ data:
   source:
     type: local
     location: ${OSH_INFRA_PATH}
-    subpath: fluent-logging
+    subpath: fluentd
     reference: master
   dependencies:
     - helm-toolkit

tools/deployment/common/fluentbit.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Lint and package chart
+make fluentbit
+
+helm upgrade --install fluentbit ./fluentbit \
+    --namespace=osh-infra
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh osh-infra
+
+#NOTE: Validate Deployment info
+helm status fluentbit

tools/deployment/multinode/125-fluentbit.sh

@@ -0,0 +1 @@
+../common/fluentbit.sh

tools/deployment/multinode/130-fluentd.sh

@@ -17,10 +17,10 @@
 set -xe
 
 #NOTE: Lint and package chart
-make fluent-logging
+make fluentd
 
 if [ ! -d "/var/log/journal" ]; then
-tee /tmp/fluent-logging.yaml << EOF
+tee /tmp/fluentd.yaml << EOF
 monitoring:
   prometheus:
     enabled: true
@@ -36,20 +36,16 @@ pod:
           - name: runlog
             mountPath: /run/log
 EOF
-helm upgrade --install fluent-logging ./fluent-logging \
+helm upgrade --install fluentd ./fluentd \
     --namespace=osh-infra \
-    --values=/tmp/fluent-logging.yaml
+    --values=/tmp/fluentd.yaml
 else
-helm upgrade --install fluent-logging ./fluent-logging \
+helm upgrade --install fluentd ./fluentd \
-    --namespace=osh-infra \
+    --namespace=osh-infra
-    --set monitoring.prometheus.enabled=true
 fi
 
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh osh-infra
 
 #NOTE: Validate Deployment info
-helm status fluent-logging
+helm status fluentd
-
-#NOTE: Run helm tests
-helm test fluent-logging

tools/deployment/osh-infra-logging/060-fluentbit.sh

@@ -0,0 +1 @@
+../common/fluentbit.sh

tools/deployment/osh-infra-logging/065-fluentd.sh

@@ -17,10 +17,10 @@
 set -xe
 
 #NOTE: Lint and package chart
-make fluent-logging
+make fluentd
 
 if [ ! -d "/var/log/journal" ]; then
-tee /tmp/fluent-logging.yaml << EOF
+tee /tmp/fluentd.yaml << EOF
 monitoring:
   prometheus:
     enabled: true
@@ -38,11 +38,11 @@ pod:
           - name: runlog
             mountPath: /run/log
 EOF
-helm upgrade --install fluent-logging ./fluent-logging \
+helm upgrade --install fluentd ./fluentd \
     --namespace=osh-infra \
-    --values=/tmp/fluent-logging.yaml
+    --values=/tmp/fluentd.yaml
 else
-helm upgrade --install fluent-logging ./fluent-logging \
+helm upgrade --install fluentd ./fluentd \
     --namespace=osh-infra \
     --set pod.replicas.fluentd=1 \
     --set monitoring.prometheus.enabled=true
@@ -52,6 +52,4 @@ fi
 ./tools/deployment/common/wait-for-pods.sh osh-infra
 
 #NOTE: Validate Deployment info
-helm status fluent-logging
+helm status fluentd
-
-helm test fluent-logging

zuul.d/jobs.yaml

@@ -60,7 +60,8 @@
         - ./tools/deployment/multinode/110-nagios.sh
         - ./tools/deployment/multinode/115-radosgw-osh-infra.sh
         - ./tools/deployment/multinode/120-elasticsearch.sh
-        - ./tools/deployment/multinode/130-fluent-logging.sh
+        - ./tools/deployment/multinode/125-fluentbit.sh
+        - ./tools/deployment/multinode/130-fluentd.sh
         - ./tools/deployment/multinode/140-kibana.sh
         - ./tools/deployment/multinode/600-grafana-selenium.sh
         - ./tools/deployment/multinode/610-nagios-selenium.sh
@@ -127,7 +128,8 @@
         - ./tools/deployment/osh-infra-logging/040-ldap.sh
         - ./tools/deployment/osh-infra-logging/050-elasticsearch.sh
         - ./tools/deployment/osh-infra-logging/055-elasticsearch-ldap.sh
-        - ./tools/deployment/osh-infra-logging/060-fluent-logging.sh
+        - ./tools/deployment/osh-infra-logging/060-fluentbit.sh
+        - ./tools/deployment/osh-infra-logging/065-fluentd.sh
         - ./tools/deployment/osh-infra-logging/070-kibana.sh
         - ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh