openstack/openstack-helm-infra
Code Issues Proposed changes

Merge "Separate fluentbit and fluentd charts"

Browse Source
This commit is contained in:
Zuul 2019-05-24 17:34:27 +00:00 committed by Gerrit Code Review
commit 5cb47aed80
40 changed files with 688 additions and 391 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
doc/source/install/multinode.rst
View File

@ -210,10 +210,10 @@ Alternatively, this step can be performed by running the script directly:
./tools/deployment/multinode/120-elasticsearch.sh
Deploy Fluent-Logging
^^^^^^^^^^^^^^^^^^^^^
Deploy Fluentbit
^^^^^^^^^^^^^^^^
.. literalinclude:: ../../../tools/deployment/multinode/130-fluent-logging.sh
.. literalinclude:: ../../../tools/deployment/multinode/125-fluentbit.sh
:language: shell
:lines: 1,17-
@ -221,4 +221,17 @@ Alternatively, this step can be performed by running the script directly:
.. code-block:: shell
./tools/deployment/multinode/130-fluent-logging.sh
./tools/deployment/multinode/125-fluentbit.sh
Deploy Fluentd
^^^^^^^^^^^^^^
.. literalinclude:: ../../../tools/deployment/multinode/130-fluentd.sh
:language: shell
:lines: 1,17-
Alternatively, this step can be performed by running the script directly:
.. code-block:: shell
./tools/deployment/multinode/130-fluentd.sh

0
fluent-logging/templates/bin/_create_template.sh.tpl → elasticsearch/templates/bin/_create_template.sh.tpl
View File

19
elasticsearch/templates/bin/_helm-tests.sh.tpl
View File

@ -52,6 +52,24 @@ function check_snapshot_repositories () {
{{ end }}
}
{{ if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }}
# Tests whether elasticsearch has successfully generated the elasticsearch index mapping
# templates defined by values.yaml
function check_templates () {
{{ range $template, $fields := .Values.conf.templates }}
{{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \
| python -c "import sys, json; print len(json.load(sys.stdin))")
if [ "${{$template}}_total_hits" -gt 0 ]; then
echo "PASS: Successful hits on {{$template}} template!"
else
echo "FAIL: No hits on query for {{$template}} template! Exiting";
exit 1;
fi
{{ end }}
}
{{ end }}
function remove_test_index () {
echo "Deleting index created for service testing"
curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
@ -63,4 +81,5 @@ create_test_index
{{ if .Values.conf.elasticsearch.snapshots.enabled }}
check_snapshot_repositories
{{ end }}
check_templates
remove_test_index

2
elasticsearch/templates/configmap-bin-elasticsearch.yaml
View File

@ -38,6 +38,8 @@ data:
{{ tuple "bin/_register-repository.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
es-cluster-wait.sh: |
{{ tuple "bin/_es-cluster-wait.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
create_template.sh: |
{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
image-repo-sync.sh: |
{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
{{- end }}

22
fluent-logging/templates/monitoring/prometheus/exporter-network-policy.yaml → elasticsearch/templates/configmap-etc-templates.yaml
View File

@ -1,11 +1,11 @@
{{/*
Copyright 2017-2018 The Openstack-Helm Authors.
Copyright 2019 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@ -14,7 +14,17 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.monitoring.prometheus.network_policy_exporter .Values.monitoring.prometheus.enabled -}}
{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-fluentd-exporter" }}
{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
{{- end -}}
{{- if .Values.manifests.configmap_etc_templates }}
{{- $envAll := . }}
---
apiVersion: v1
kind: Secret
metadata:
name: elasticsearch-templates-etc
type: Opaque
data:
{{ range $template, $fields := .Values.conf.templates }}
{{ $template }}.json: {{ toJson $fields | b64enc }}
{{ end }}
{{- end }}

38
fluent-logging/templates/job-elasticsearch-template.yaml → elasticsearch/templates/job-elasticsearch-template.yaml
View File

@ -14,26 +14,26 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }}
{{- if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }}
{{- $envAll := . }}
{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
{{- $mounts_elasticsearch_template := .Values.pod.mounts.elasticsearch_template.elasticsearch_template }}
{{- $mounts_elasticsearch_template_init := .Values.pod.mounts.elasticsearch_template.init_container }}
{{- $mounts_elasticsearch_templates := .Values.pod.mounts.elasticsearch_templates.elasticsearch_templates }}
{{- $mounts_elasticsearch_templates_init := .Values.pod.mounts.elasticsearch_templates.init_container }}
{{- $serviceAccountName := "fluent-logging-elasticsearch-template" }}
{{ tuple $envAll "elasticsearch_template" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
{{- $serviceAccountName := "create-elasticsearch-templates" }}
{{ tuple $envAll "elasticsearch_templates" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: elasticsearch-template
name: create-elasticsearch-templates
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
spec:
template:
metadata:
labels:
{{ tuple $envAll "fluentd" "elasticsearch-template" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
{{ tuple $envAll "elasticsearch" "create-templates" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
spec:
{{ dict "envAll" $envAll "application" "elasticsearch_template" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
serviceAccountName: {{ $serviceAccountName }}
@ -41,11 +41,11 @@ spec:
nodeSelector:
{{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }}
initContainers:
{{ tuple $envAll "elasticsearch_template" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{ tuple $envAll "elasticsearch_templates" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: elasticsearch-template
{{ tuple $envAll "elasticsearch_template" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_template | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
- name: create-elasticsearch-templates
{{ tuple $envAll "elasticsearch_templates" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_templates | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "elasticsearch_template" "container" "elasticsearch_template" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: ELASTICSEARCH_HOST
@ -67,27 +67,27 @@ spec:
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: fluent-logging-bin
- name: elasticsearch-bin
mountPath: /tmp/create_template.sh
subPath: create_template.sh
readOnly: true
{{ range $template, $fields := .Values.conf.templates }}
- name: fluent-logging-etc
- name: elasticsearch-templates-etc
mountPath: /tmp/{{$template}}.json
subPath: {{$template}}.json
readOnly: true
{{ end }}
{{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 12 }}{{ end }}
{{ if $mounts_elasticsearch_templates.volumeMounts }}{{ toYaml $mounts_elasticsearch_templates.volumeMounts | indent 12 }}{{ end }}
volumes:
- name: pod-tmp
emptyDir: {}
- name: fluent-logging-bin
- name: elasticsearch-bin
configMap:
name: fluent-logging-bin
name: elasticsearch-bin
defaultMode: 0555
- name: fluent-logging-etc
- name: elasticsearch-templates-etc
secret:
secretName: fluent-logging-etc
secretName: elasticsearch-templates-etc
defaultMode: 0444
{{ if $mounts_elasticsearch_template.volumes }}{{ toYaml $mounts_elasticsearch_template.volumes | indent 8 }}{{ end }}
{{ if $mounts_elasticsearch_templates.volumes }}{{ toYaml $mounts_elasticsearch_templates.volumes | indent 8 }}{{ end }}
{{- end }}

45
elasticsearch/values.yaml
View File

@ -30,6 +30,7 @@ images:
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
snapshot_repository: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_xenial
es_cluster_wait: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_xenial
elasticsearch_templates: docker.io/openstackhelm/heat:newton
image_repo_sync: docker.io/docker:17.07.0
pull_policy: "IfNotPresent"
local_registry:
@ -82,6 +83,10 @@ dependencies:
services:
- endpoint: internal
service: elasticsearch
elasticsearch_templates:
services: null
jobs:
- elasticsearch-cluster-wait
image_repo_sync:
services:
- endpoint: internal
@ -220,6 +225,8 @@ pod:
mounts:
elasticsearch:
elasticsearch:
elasticsearch_templates:
elasticsearch_templates:
resources:
enabled: false
apache_proxy:
@ -265,6 +272,13 @@ pod:
limits:
memory: "1024Mi"
cpu: "2000m"
elasticsearch_templates:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "1024Mi"
cpu: "2000m"
image_repo_sync:
requests:
memory: "128Mi"
@ -635,6 +649,35 @@ conf:
es:
all: true
timeout: 20s
templates:
fluent:
template: "logstash-*"
index_patterns: "logstash-*"
settings:
number_of_shards: 1
mappings:
fluent:
properties:
kubernetes:
properties:
container_name:
type: keyword
index: false
docker_id:
type: keyword
index: false
host:
type: keyword
index: false
namespace_name:
type: keyword
index: false
pod_id:
type: keyword
index: false
pod_name:
type: keyword
index: false
endpoints:
cluster_domain_suffix: cluster.local
@ -771,11 +814,13 @@ manifests:
configmap_bin_elasticsearch: true
configmap_etc_curator: true
configmap_etc_elasticsearch: true
configmap_etc_templates: true
cron_curator: true
deployment_client: true
deployment_master: true
ingress: true
job_cluster_wait: true
job_elasticsearch_templates: true
job_image_repo_sync: true
job_snapshot_repository: true
job_s3_user: true

30
fluent-logging/README.rst
View File

@ -1,30 +0,0 @@
Fluentd-logging
===============
OpenStack-Helm defines a centralized logging mechanism to provide insight into
the state of the OpenStack services and infrastructure components as
well as underlying kubernetes platform. Among the requirements for a logging
platform, where log data can come from and where log data need to be delivered
are very variable. To support various logging scenarios, OpenStack-Helm should
provide a flexible mechanism to meet with certain operation needs. This chart
proposes fast and lightweight log forwarder and full featured log aggregator
complementing each other providing a flexible and reliable solution. Especially,
Fluent-bit is proposed as a log forwarder and Fluentd is proposed as a main log
aggregator and processor.
Mechanism
---------
Fluent-bit, Fluentd meet OpenStack-Helm's logging requirements for gathering,
aggregating, and delivering of logged events. Flunt-bit runs as a daemonset on
each node and mounts the /var/lib/docker/containers directory. The Docker
container runtime engine directs events posted to stdout and stderr to this
directory on the host. Fluent-bit then forward the contents of that directory to
Fluentd. Fluentd runs as deployment at the designated nodes and expose service
for Fluent-bit to foward logs. Fluentd should then apply the Logstash format to
the logs. Fluentd can also write kubernetes and OpenStack metadata to the logs.
Fluentd will then forward the results to Elasticsearch and to optionally kafka.
Elasticsearch indexes the logs in a logstash-* index by default. kafka stores
the logs in a 'logs' topic by default. Any external tool can then consume the
'logs' topic.

41
fluent-logging/templates/bin/_helm-tests.sh.tpl
View File

@ -1,41 +0,0 @@
#!/bin/bash
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
{{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }}
# Tests whether fluent-logging has successfully generated the elasticsearch index mapping
# templates defined by values.yaml
function check_templates () {
{{ range $template, $fields := .Values.conf.templates }}
{{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \
| python -c "import sys, json; print len(json.load(sys.stdin))")
if [ "${{$template}}_total_hits" -gt 0 ]; then
echo "PASS: Successful hits on {{$template}} template, provided by fluent-logging!"
else
echo "FAIL: No hits on query for {{$template}} template! Exiting";
exit 1;
fi
{{ end }}
}
{{ end }}
{{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }}
check_templates
{{ end }}

75
fluent-logging/templates/pod-helm-tests.yaml
View File

@ -1,75 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.helm_tests }}
{{- $envAll := . }}
{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
{{- $serviceAccountName := print .Release.Name "-test" }}
{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: v1
kind: Pod
metadata:
name: "{{.Release.Name}}-test"
labels:
{{ tuple $envAll "fluentd" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
annotations:
"helm.sh/hook": test-success
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
spec:
{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }}
serviceAccountName: {{ $serviceAccountName }}
nodeSelector:
{{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }}
restartPolicy: Never
initContainers:
{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }}
containers:
- name: {{.Release.Name}}-helm-tests
{{ tuple $envAll "helm_tests" | include "helm-toolkit.snippets.image" | indent 6 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }}
{{ dict "envAll" $envAll "application" "test" "container" "helm_tests" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }}
command:
- /tmp/helm-tests.sh
env:
- name: ELASTICSEARCH_USERNAME
valueFrom:
secretKeyRef:
name: {{ $esUserSecret }}
key: ELASTICSEARCH_USERNAME
- name: ELASTICSEARCH_PASSWORD
valueFrom:
secretKeyRef:
name: {{ $esUserSecret }}
key: ELASTICSEARCH_PASSWORD
- name: ELASTICSEARCH_ENDPOINT
value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: fluent-logging-bin
mountPath: /tmp/helm-tests.sh
subPath: helm-tests.sh
readOnly: true
volumes:
- name: pod-tmp
emptyDir: {}
- name: fluent-logging-bin
configMap:
name: fluent-logging-bin
defaultMode: 0555
{{- end }}

24
fluentbit/Chart.yaml Normal file
View File

@ -0,0 +1,24 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
description: OpenStack-Helm Fluentbit
name: fluentbit
version: 0.1.0
home: http://www.fluentbit.io/
sources:
- https://github.com/fluent/fluentbit
- https://git.openstack.org/cgit/openstack/openstack-helm-infra
maintainers:
- name: OpenStack-Helm Authors

0
fluent-logging/requirements.yaml → fluentbit/requirements.yaml
View File

0
fluent-logging/templates/bin/_fluent-bit.sh.tpl → fluentbit/templates/bin/_fluent-bit.sh.tpl
View File

8
fluent-logging/templates/configmap-bin.yaml → fluentbit/templates/configmap-bin.yaml
View File

@ -20,16 +20,10 @@ limitations under the License.
apiVersion: v1
kind: ConfigMap
metadata:
name: fluent-logging-bin
name: fluentbit-bin
data:
fluentd.sh: |
{{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
fluent-bit.sh: |
{{ tuple "bin/_fluent-bit.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
helm-tests.sh: |
{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
create_template.sh: |
{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
image-repo-sync.sh: |
{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
{{- end }}

6
fluent-logging/templates/configmap-etc.yaml → fluentbit/templates/configmap-etc.yaml
View File

@ -20,13 +20,9 @@ limitations under the License.
apiVersion: v1
kind: Secret
metadata:
name: fluent-logging-etc
name: fluentbit-etc
type: Opaque
data:
fluent-bit.conf: {{ .Values.conf.fluentbit.template | b64enc }}
parsers.conf: {{ .Values.conf.parsers.template | b64enc }}
fluent.conf: {{ .Values.conf.fluentd.template | b64enc }}
{{ range $template, $fields := .Values.conf.templates }}
{{ $template }}.json: {{ toJson $fields | b64enc }}
{{ end }}
{{- end }}

18
fluent-logging/templates/daemonset-fluent-bit.yaml → fluentbit/templates/daemonset-fluent-bit.yaml
View File

@ -95,7 +95,7 @@ spec:
configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
{{ dict "envAll" $envAll "podName" "fluentbit" "containerNames" (list "fluentbit") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
spec:
{{ dict "envAll" $envAll "application" "daemon" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
{{ dict "envAll" $envAll "application" "fluentbit" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
serviceAccountName: {{ $serviceAccountName }}
{{ if $envAll.Values.pod.tolerations.fluentbit.enabled }}
{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
@ -112,7 +112,7 @@ spec:
- name: fluentbit
{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "daemon" "container" "fluentbit" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
{{ dict "envAll" $envAll "application" "fluentbit" "container" "fluentbit" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
command:
- /tmp/fluent-bit.sh
env:
@ -123,7 +123,7 @@ spec:
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: fluent-logging-bin
- name: fluentbit-bin
mountPath: /tmp/fluent-bit.sh
subPath: fluent-bit.sh
readOnly: true
@ -133,11 +133,11 @@ spec:
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: fluent-logging-etc
- name: fluentbit-etc
mountPath: /fluent-bit/etc/fluent-bit.conf
subPath: fluent-bit.conf
readOnly: true
- name: fluent-logging-etc
- name: fluentbit-etc
mountPath: /fluent-bit/etc/parsers.conf
subPath: parsers.conf
readOnly: true
@ -151,13 +151,13 @@ spec:
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: fluent-logging-bin
- name: fluentbit-bin
configMap:
name: fluent-logging-bin
name: fluentbit-bin
defaultMode: 0555
- name: fluent-logging-etc
- name: fluentbit-etc
secret:
secretName: fluent-logging-etc
secretName: fluentbit-etc
defaultMode: 0444
{{ if $mounts_fluentbit.volumes }}{{ toYaml $mounts_fluentbit.volumes | indent 8 }}{{ end }}
{{- end }}

2
fluent-logging/templates/job-image-repo-sync.yaml → fluentbit/templates/job-image-repo-sync.yaml
View File

@ -15,6 +15,6 @@ limitations under the License.
*/}}
{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }}
{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluent-logging" -}}
{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluentbit" -}}
{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }}
{{- end }}

256
fluentbit/values.yaml Normal file
View File

@ -0,0 +1,256 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Default values for fluentbit
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
release_group: null
labels:
fluentbit:
node_selector_key: openstack-control-plane
node_selector_value: enabled
images:
tags:
fluentbit: docker.io/fluent/fluent-bit:0.14.2
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
image_repo_sync: docker.io/docker:17.07.0
pull_policy: IfNotPresent
local_registry:
active: false
exclude:
- dep_check
- image_repo_sync
dependencies:
dynamic:
common:
local_image_registry:
jobs:
- fluentbit-image-repo-sync
services:
- endpoint: node
service: local_image_registry
static:
image_repo_sync:
services:
- endpoint: internal
service: local_image_registry
conf:
fluentbit:
template: |
[SERVICE]
Daemon false
Flush 30
Log_Level info
Parsers_File parsers.conf
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name tail
Path /var/log/kern.log
Tag kernel
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name tail
Parser docker
Path /var/log/containers/*.log
Tag kube.*
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name tail
Path /var/log/libvirt/libvirtd.log
Tag libvirt
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name tail
Path /var/log/libvirt/qemu/*.log
Tag qemu
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name systemd
Path ${JOURNAL_PATH}
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
Tag journal.*
[INPUT]
Buffer_Chunk_Size 1M
Buffer_Max_Size 1M
Mem_Buf_Limit 5MB
Name systemd
Path ${JOURNAL_PATH}
Systemd_Filter _SYSTEMD_UNIT=docker.service
Tag journal.*
[FILTER]
Interval 1s
Match **
Name throttle
Rate 1000
Window 300
[FILTER]
Match libvirt
Name record_modifier
Record hostname ${HOSTNAME}
[FILTER]
Match qemu
Name record_modifier
Record hostname ${HOSTNAME}
[FILTER]
Match kernel
Name record_modifier
Record hostname ${HOSTNAME}
[FILTER]
Match journal.**
Name modify
Rename _BOOT_ID BOOT_ID
Rename _CAP_EFFECTIVE CAP_EFFECTIVE
Rename _CMDLINE CMDLINE
Rename _COMM COMM
Rename _EXE EXE
Rename _GID GID
Rename _HOSTNAME HOSTNAME
Rename _MACHINE_ID MACHINE_ID
Rename _PID PID
Rename _SYSTEMD_CGROUP SYSTEMD_CGROUP
Rename _SYSTEMD_SLICE SYSTEMD_SLICE
Rename _SYSTEMD_UNIT SYSTEMD_UNIT
Rename _TRANSPORT TRANSPORT
Rename _UID UID
[OUTPUT]
Match **.fluentd**
Name null
[FILTER]
Match kube.*
Merge_JSON_Log true
Name kubernetes
[OUTPUT]
Host ${FLUENTD_HOST}
Match *
Name forward
Port ${FLUENTD_PORT}
parsers:
template: |
[PARSER]
Decode_Field_As escaped_utf8 log
Format json
Name docker
Time_Format %Y-%m-%dT%H:%M:%S.%L
Time_Keep true
Time_Key time
endpoints:
cluster_domain_suffix: cluster.local
local_image_registry:
name: docker-registry
namespace: docker-registry
hosts:
default: localhost
internal: docker-registry
node: localhost
host_fqdn_override:
default: null
port:
registry:
node: 5000
fluentd:
namespace: null
name: fluentd
hosts:
default: fluentd-logging
host_fqdn_override:
default: null
path:
default: null
scheme:
default: http
port:
service:
default: 24224
metrics:
default: 24220
pod:
security_context:
fluentbit:
pod:
runAsUser: 65534
container:
fluentbit:
runAsUser: 0
readOnlyRootFilesystem: false
affinity:
anti:
type:
default: preferredDuringSchedulingIgnoredDuringExecution
topologyKey:
default: kubernetes.io/hostname
lifecycle:
upgrades:
daemonsets:
pod_replacement_strategy: RollingUpdate
fluentbit:
enabled: true
min_ready_seconds: 0
max_unavailable: 1
resources:
enabled: false
fluentbit:
limits:
memory: '400Mi'
cpu: '400m'
requests:
memory: '100Mi'
cpu: '100m'
tolerations:
fluentbit:
enabled: false
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
- key: node-role.kubernetes.io/node
operator: Exists
mounts:
fluentbit:
fluentbit:
manifests:
configmap_bin: true
configmap_etc: true
daemonset_fluentbit: true
job_image_repo_sync: true

3
fluent-logging/Chart.yaml → fluentd/Chart.yaml
View File

@ -14,11 +14,10 @@
apiVersion: v1
description: OpenStack-Helm Fluentd
name: fluent-logging
name: fluentd
version: 0.1.0
home: http://www.fluentbit.io/
sources:
- https://github.com/fluent/fluentbit
- https://github.com/fluent/fluentd
- https://git.openstack.org/cgit/openstack/openstack-helm-infra
maintainers:

18
fluentd/requirements.yaml Normal file
View File

@ -0,0 +1,18 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
dependencies:
- name: helm-toolkit
repository: http://localhost:8879/charts/
version: 0.1.0

0
fluent-logging/templates/bin/_fluentd.sh.tpl → fluentd/templates/bin/_fluentd.sh.tpl
View File

29
fluentd/templates/configmap-bin.yaml Normal file
View File

@ -0,0 +1,29 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.configmap_bin }}
{{- $envAll := . }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: fluentd-bin
data:
fluentd.sh: |
{{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
image-repo-sync.sh: |
{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
{{- end }}

27
fluentd/templates/configmap-etc.yaml Normal file
View File

@ -0,0 +1,27 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.configmap_etc }}
{{- $envAll := . }}
---
apiVersion: v1
kind: Secret
metadata:
name: fluentd-etc
type: Opaque
data:
fluent.conf: {{ .Values.conf.fluentd.template | b64enc }}
{{- end }}

16
fluent-logging/templates/deployment-fluentd.yaml → fluentd/templates/deployment-fluentd.yaml
View File

@ -95,7 +95,7 @@ spec:
configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
spec:
{{ dict "envAll" $envAll "application" "internal" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
{{ dict "envAll" $envAll "application" "fluentd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
serviceAccountName: {{ $serviceAccountName }}
affinity:
{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
@ -108,7 +108,7 @@ spec:
- name: fluentd
{{ tuple $envAll "fluentd" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.fluentd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "internal" "container" "fluentd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
{{ dict "envAll" $envAll "application" "fluentd" "container" "fluentd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
command:
- /tmp/fluentd.sh
- start
@ -149,11 +149,11 @@ spec:
mountPath: /tmp
- name: pod-etc-fluentd
mountPath: /fluentd/etc
- name: fluent-logging-etc
- name: fluentd-etc
mountPath: /fluentd/etc/fluent.conf
subPath: fluent.conf
readOnly: true
- name: fluent-logging-bin
- name: fluentd-bin
mountPath: /tmp/fluentd.sh
subPath: fluentd.sh
readOnly: true
@ -163,13 +163,13 @@ spec:
emptyDir: {}
- name: pod-etc-fluentd
emptyDir: {}
- name: fluent-logging-etc
- name: fluentd-etc
secret:
secretName: fluent-logging-etc
secretName: fluentd-etc
defaultMode: 0444
- name: fluent-logging-bin
- name: fluentd-bin
configMap:
name: fluent-logging-bin
name: fluentd-bin
defaultMode: 0555
{{- if $mounts_fluentd.volumes }}{{ toYaml $mounts_fluentd.volumes | indent 8 }}{{- end }}
{{- end }}

20
fluentd/templates/job-image-repo-sync.yaml Normal file
View File

@ -0,0 +1,20 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }}
{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluentd" -}}
{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }}
{{- end }}

0
fluent-logging/templates/monitoring/prometheus/bin/_fluentd-exporter.sh.tpl → fluentd/templates/monitoring/prometheus/bin/_fluentd-exporter.sh.tpl
View File

0
fluent-logging/templates/monitoring/prometheus/exporter-configmap-bin.yaml → fluentd/templates/monitoring/prometheus/exporter-configmap-bin.yaml
View File

0
fluent-logging/templates/monitoring/prometheus/exporter-deployment.yaml → fluentd/templates/monitoring/prometheus/exporter-deployment.yaml
View File

0
fluent-logging/templates/monitoring/prometheus/exporter-service.yaml → fluentd/templates/monitoring/prometheus/exporter-service.yaml
View File

0
fluent-logging/templates/network-policy.yaml → fluentd/templates/network_policy.yaml
View File

0
fluent-logging/templates/secret-elasticsearch-creds.yaml → fluentd/templates/secret-elasticsearch-creds.yaml
View File

0
fluent-logging/templates/service-fluentd.yaml → fluentd/templates/service-fluentd.yaml
View File

154
fluent-logging/values.yaml → fluentd/values.yaml
View File

@ -22,22 +22,12 @@ labels:
fluentd:
node_selector_key: openstack-control-plane
node_selector_value: enabled
fluentbit:
node_selector_key: openstack-control-plane
node_selector_value: enabled
prometheus_fluentd_exporter:
node_selector_key: openstack-control-plane
node_selector_value: enabled
job:
node_selector_key: openstack-control-plane
node_selector_value: enabled
test:
node_selector_key: openstack-control-plane
node_selector_value: enabled
images:
tags:
fluentbit: docker.io/fluent/fluent-bit:0.14.2
fluentd: docker.io/fluent/fluentd-kubernetes-daemonset:v1.2-debian-elasticsearch
prometheus_fluentd_exporter: docker.io/bitnami/fluentd-exporter:0.2.0
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
@ -60,33 +50,13 @@ dependencies:
common:
local_image_registry:
jobs:
- fluent-logging-image-repo-sync
- fluentd-image-repo-sync
services:
- endpoint: node
service: local_image_registry
static:
elasticsearch_template:
services:
- endpoint: internal
service: elasticsearch
fluentbit:
jobs:
- elasticsearch-template
services:
- endpoint: internal
service: fluentd
fluentd:
jobs:
- elasticsearch-template
services:
- endpoint: internal
service: elasticsearch
fluentd_with_kafka:
services:
- endpoint: internal
service: elasticsearch
- endpoint: public
service: kafka
services: null
image_repo_sync:
services:
- endpoint: internal
@ -95,12 +65,6 @@ dependencies:
services:
- endpoint: internal
service: fluentd
tests:
services:
- endpoint: internal
service: elasticsearch
- endpoint: internal
service: fluentd
conf:
fluentd:
@ -436,40 +400,10 @@ conf:
Time_Format %Y-%m-%dT%H:%M:%S.%L
Time_Keep true
Time_Key time
fluentd_exporter:
log:
format: "logger:stdout?json=true"
level: "info"
templates:
fluent:
template: "logstash-*"
index_patterns: "logstash-*"
settings:
number_of_shards: 1
mappings:
fluent:
properties:
kubernetes:
properties:
container_name:
type: keyword
index: false
docker_id:
type: keyword
index: false
host:
type: keyword
index: false
namespace_name:
type: keyword
index: false
pod_id:
type: keyword
index: false
pod_name:
type: keyword
index: false
endpoints:
cluster_domain_suffix: cluster.local
@ -506,21 +440,6 @@ endpoints:
port:
http:
default: 80
kafka:
namespace: null
name: kafka
hosts:
default: kafka-logging
public: kafka
host_fqdn_override:
default: null
path:
default: null
scheme:
default: http
port:
service:
default: 9092
fluentd:
namespace: null
name: fluentd
@ -577,34 +496,13 @@ network_policy:
pod:
security_context:
daemon:
pod:
runAsUser: 65534
container:
fluentbit:
runAsUser: 0
readOnlyRootFilesystem: true
internal:
fluentd:
pod:
runAsUser: 65534
container:
fluentd:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
elasticsearch_template:
pod:
runAsUser: 65534
container:
elasticsearch_template:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
test:
pod:
runAsUser: 65534
container:
helm_test:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
exporter:
pod:
runAsUser: 65534
@ -622,12 +520,6 @@ pod:
default: 10
lifecycle:
upgrades:
daemonsets:
pod_replacement_strategy: RollingUpdate
fluentbit:
enabled: true
min_ready_seconds: 0
max_unavailable: 1
deployments:
revision_history: 3
pod_replacement_strategy: RollingUpdate
@ -644,13 +536,6 @@ pod:
prometheus_fluentd_exporter: 1
resources:
enabled: false
fluentbit:
limits:
memory: '400Mi'
cpu: '400m'
requests:
memory: '100Mi'
cpu: '100m'
fluentd:
limits:
memory: '1024Mi'
@ -665,47 +550,15 @@ pod:
requests:
memory: "128Mi"
cpu: "500m"
jobs:
image_repo_sync:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "1024Mi"
cpu: "2000m"
tests:
requests:
memory: '128Mi'
cpu: '100m'
limits:
memory: '1024Mi'
cpu: '2000m'
tolerations:
fluentbit:
enabled: false
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
- key: node-role.kubernetes.io/node
operator: Exists
mounts:
fluentd:
fluentd:
fluentbit:
fluentbit:
fluent_tests:
fluent_tests:
elasticsearch_template:
init_container:
elasticsearch_template:
manifests:
configmap_bin: true
configmap_etc: true
deployment_fluentd: true
daemonset_fluentbit: true
job_image_repo_sync: true
helm_tests: true
monitoring:
prometheus:
configmap_bin_exporter: true
@ -715,4 +568,3 @@ manifests:
network_policy: false
secret_elasticsearch: true
service_fluentd: true
job_elasticsearch_template: true

136
tools/deployment/armada/manifests/armada-lma.yaml
View File

@ -328,6 +328,81 @@ data:
master: "-Xms512m -Xmx512m"
snapshots:
enabled: true
templates:
syslog:
template: "syslog-*"
index_patterns: "syslog-*"
settings:
number_of_shards: 1
mappings:
syslog:
properties:
cluster:
type: keyword
app:
type: keyword
pid:
type: integer
host:
type: keyword
log:
type: text
oslo_openstack_fluentd:
template: "openstack-*"
index_patterns: "openstack-*"
settings:
number_of_shards: 1
mappings:
oslo_openstack_fluentd:
properties:
extra:
properties:
project:
type: text
norms: false
version:
type: text
norms: false
filename:
type: text
norms: false
funcname:
type: text
norms: false
message:
type: text
norms: false
process_name:
type: keyword
index: false
docker_fluentd:
template: "logstash-*"
index_patterns: "logstash-*"
settings:
number_of_shards: 1
mappings:
docker_fluentd:
properties:
kubernetes:
properties:
container_name:
type: keyword
index: false
docker_id:
type: keyword
index: false
host:
type: keyword
index: false
namespace_name:
type: keyword
index: false
pod_id:
type: keyword
index: false
pod_name:
type: keyword
index: false
source:
type: local
location: ${OSH_INFRA_PATH}
@ -339,19 +414,17 @@ data:
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: fluent-logging
name: fluentbit
data:
chart_name: fluent-logging
release: fluent-logging
chart_name: fluentbit
release: fluentbit
namespace: osh-infra
wait:
timeout: 3600
labels:
release_group: osh-infra-fluent-logging
release_group: osh-infra-fluentbit
resources:
- type: daemonset
- type: deployment
- type: job
test:
timeout: 600
install:
@ -362,10 +435,52 @@ data:
delete:
- type: job
labels:
release_group: osh-infra-fluent-logging
release_group: osh-infra-fluentbit
- type: pod
labels:
release_group: osh-infra-fluent-logging
release_group: osh-infra-fluentbit
component: test
values:
release_uuid: ${RELEASE_UUID}
fluentbit:
node_selector_key: openstack-control-plane
node_selector_value: enabled
source:
type: local
location: ${OSH_INFRA_PATH}
subpath: fluentbit
reference: master
dependencies:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: fluentd
data:
chart_name: fluentd
release: fluentd
namespace: osh-infra
wait:
timeout: 3600
labels:
release_group: osh-infra-fluentd
resources:
- type: deployment
test:
timeout: 600
install:
no_hooks: False
upgrade:
no_hooks: False
pre:
delete:
- type: job
labels:
release_group: osh-infra-fluentd
- type: pod
labels:
release_group: osh-infra-fluentd
component: test
values:
release_uuid: ${RELEASE_UUID}
@ -384,9 +499,6 @@ data:
fluentd:
node_selector_key: openstack-control-plane
node_selector_value: enabled
fluentbit:
node_selector_key: openstack-control-plane
node_selector_value: enabled
prometheus_fluentd_exporter:
node_selector_key: openstack-control-plane
node_selector_value: enabled
@ -396,7 +508,7 @@ data:
source:
type: local
location: ${OSH_INFRA_PATH}
subpath: fluent-logging
subpath: fluentd
reference: master
dependencies:
- helm-toolkit

29
tools/deployment/common/fluentbit.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Lint and package chart
make fluentbit
helm upgrade --install fluentbit ./fluentbit \
--namespace=osh-infra
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh osh-infra
#NOTE: Validate Deployment info
helm status fluentbit

1
tools/deployment/multinode/125-fluentbit.sh Symbolic link
View File

@ -0,0 +1 @@
../common/fluentbit.sh

18
tools/deployment/multinode/130-fluent-logging.sh → tools/deployment/multinode/130-fluentd.sh
View File

@ -17,10 +17,10 @@
set -xe
#NOTE: Lint and package chart
make fluent-logging
make fluentd
if [ ! -d "/var/log/journal" ]; then
tee /tmp/fluent-logging.yaml << EOF
tee /tmp/fluentd.yaml << EOF
monitoring:
prometheus:
enabled: true
@ -36,20 +36,16 @@ pod:
- name: runlog
mountPath: /run/log
EOF
helm upgrade --install fluent-logging ./fluent-logging \
helm upgrade --install fluentd ./fluentd \
--namespace=osh-infra \
--values=/tmp/fluent-logging.yaml
--values=/tmp/fluentd.yaml
else
helm upgrade --install fluent-logging ./fluent-logging \
--namespace=osh-infra \
--set monitoring.prometheus.enabled=true
helm upgrade --install fluentd ./fluentd \
--namespace=osh-infra
fi
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh osh-infra
#NOTE: Validate Deployment info
helm status fluent-logging
#NOTE: Run helm tests
helm test fluent-logging
helm status fluentd

1
tools/deployment/osh-infra-logging/060-fluentbit.sh Symbolic link
View File

@ -0,0 +1 @@
../common/fluentbit.sh

14
tools/deployment/osh-infra-logging/060-fluent-logging.sh → tools/deployment/osh-infra-logging/065-fluentd.sh
View File

@ -17,10 +17,10 @@
set -xe
#NOTE: Lint and package chart
make fluent-logging
make fluentd
if [ ! -d "/var/log/journal" ]; then
tee /tmp/fluent-logging.yaml << EOF
tee /tmp/fluentd.yaml << EOF
monitoring:
prometheus:
enabled: true
@ -38,11 +38,11 @@ pod:
- name: runlog
mountPath: /run/log
EOF
helm upgrade --install fluent-logging ./fluent-logging \
helm upgrade --install fluentd ./fluentd \
--namespace=osh-infra \
--values=/tmp/fluent-logging.yaml
--values=/tmp/fluentd.yaml
else
helm upgrade --install fluent-logging ./fluent-logging \
helm upgrade --install fluentd ./fluentd \
--namespace=osh-infra \
--set pod.replicas.fluentd=1 \
--set monitoring.prometheus.enabled=true
@ -52,6 +52,4 @@ fi
./tools/deployment/common/wait-for-pods.sh osh-infra
#NOTE: Validate Deployment info
helm status fluent-logging
helm test fluent-logging
helm status fluentd

6
zuul.d/jobs.yaml
View File

@ -60,7 +60,8 @@
- ./tools/deployment/multinode/110-nagios.sh
- ./tools/deployment/multinode/115-radosgw-osh-infra.sh
- ./tools/deployment/multinode/120-elasticsearch.sh
- ./tools/deployment/multinode/130-fluent-logging.sh
- ./tools/deployment/multinode/125-fluentbit.sh
- ./tools/deployment/multinode/130-fluentd.sh
- ./tools/deployment/multinode/140-kibana.sh
- ./tools/deployment/multinode/600-grafana-selenium.sh
- ./tools/deployment/multinode/610-nagios-selenium.sh
@ -127,7 +128,8 @@
- ./tools/deployment/osh-infra-logging/040-ldap.sh
- ./tools/deployment/osh-infra-logging/050-elasticsearch.sh
- ./tools/deployment/osh-infra-logging/055-elasticsearch-ldap.sh
- ./tools/deployment/osh-infra-logging/060-fluent-logging.sh
- ./tools/deployment/osh-infra-logging/060-fluentbit.sh
- ./tools/deployment/osh-infra-logging/065-fluentd.sh
- ./tools/deployment/osh-infra-logging/070-kibana.sh
- ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh