openstack/openstack-helm-infra
Code Issues Proposed changes

Ceph: Remove fluentbit sidecars, mount hostpath for logs

Browse Source 
This removes the fluentbit sidecars from the ceph-mon and ceph-osd
charts. Instead, we mount /var/log/ceph as a hostpath, and use the
fluentbit daemonset to target the mounted log files instead

This also updates the fluentd configuration to better handle the
correct configuration type for flush_interval (time vs int), as
well as updates the fluentd elasticsearch output values to help
address the gate failures resulting from the Elasticsearch bulk
endpoints failing

Change-Id: If3f2ff6371f267ed72379de25ff463079ba4cddc
This commit is contained in:
Steve Wilkerson 2018-10-05 15:52:04 -05:00
parent b3b4e6858b
commit 92717bdc72
15 changed files with 70 additions and 330 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
ceph-mon/templates/bin/mon/_fluentbit-sidecar.sh.tpl
View File

@ -1,19 +0,0 @@
#!/bin/sh
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf

4
ceph-mon/templates/configmap-bin.yaml
View File

@ -60,8 +60,4 @@ data:
utils-checkPGs.sh: |
{{ tuple "bin/utils/_checkPGs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{ if .Values.logging.fluentd }}
fluentbit-sidecar.sh: |
{{ tuple "bin/mon/_fluentbit-sidecar.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{ end }}
{{- end }}

15
ceph-mon/templates/configmap-etc.yaml
View File

@ -42,15 +42,6 @@ limitations under the License.
{{- if empty .Values.conf.ceph.osd.public_network -}}
{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
{{- end -}}
{{- if not (has "fluentd_output" .Values.conf.fluentbit) -}}
{{- $fluentd_host := tuple "fluentd" "internal" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
{{- $fluentd_port := tuple "fluentd" "internal" "service" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $fluentd_output := dict "header" "output" "Name" "forward" "Match" "*" "Host" $fluentd_host "Port" $fluentd_port -}}
{{- $_ := set .Values "__fluentbit_config" ( list $fluentd_output) -}}
{{- $__fluentbit_config := append .Values.conf.fluentbit .Values.__fluentbit_config -}}
{{- $_ := set .Values.conf "fluentbit" $__fluentbit_config -}}
{{- end -}}
---
apiVersion: v1
kind: ConfigMap
@ -59,12 +50,6 @@ metadata:
data:
ceph.conf: |
{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
{{ if .Values.logging.fluentd }}
fluent-bit.conf: |
{{ include "ceph-mon.utils.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
parsers.conf: |
{{ include "ceph-mon.utils.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
{{ end }}
{{- end }}
{{- end }}
{{- end }}

43
ceph-mon/templates/daemonset-mon.yaml
View File

@ -91,6 +91,19 @@ spec:
- name: pod-run
mountPath: /run
readOnly: false
- name: ceph-log-ownership
{{ tuple $envAll "ceph_mon" | include "helm-toolkit.snippets.image" | indent 10 }}
securityContext:
runAsUser: 0
command:
- chown
- -R
- ceph:root
- /var/log/ceph
volumeMounts:
- name: pod-var-log
mountPath: /var/log/ceph
readOnly: false
containers:
- name: ceph-mon
{{ tuple $envAll "ceph_mon" | include "helm-toolkit.snippets.image" | indent 10 }}
@ -195,33 +208,13 @@ spec:
- name: pod-run
mountPath: /run
readOnly: false
- name: varlog
- name: pod-var-log
mountPath: /var/log/ceph
{{ if .Values.logging.fluentd }}
- name: fluentbit-sidecar
{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
command:
- /tmp/fluentbit-sidecar.sh
volumeMounts:
- name: ceph-mon-bin
mountPath: /tmp/fluentbit-sidecar.sh
subPath: fluentbit-sidecar.sh
readOnly: true
- name: varlog
mountPath: /var/log/ceph
- name: ceph-mon-etc
mountPath: /fluent-bit/etc/fluent-bit.conf
subPath: fluent-bit.conf
readOnly: true
- name: ceph-mon-etc
mountPath: /fluent-bit/etc/parsers.conf
subPath: parsers.conf
readOnly: true
{{ end }}
readOnly: false
volumes:
- name: varlog
emptyDir: {}
- name: pod-var-log
hostPath:
path: /var/log/ceph
- name: ceph-mon-bin
configMap:
name: ceph-mon-bin

38
ceph-mon/templates/utils/_to_fluentbit_conf.tpl
View File

@ -1,38 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
# This function generates fluentbit configuration files with entries in the
# ceph-mon values.yaml. It results in a configuration section with the
# following format (for as many key/value pairs defined in values for a section):
# [HEADER]
# key value
# key value
# key value
# The configuration schema can be found here:
# http://fluentbit.io/documentation/0.12/configuration/schema.html
{{- define "ceph-mon.utils.to_fluentbit_conf" -}}
{{- range $values := . -}}
{{- range $section := . -}}
{{- $header := pick . "header" -}}
{{- $config := omit . "header" }}
[{{$header.header | upper }}]
{{range $key, $value := $config -}}
{{ $key | indent 4 }} {{ $value }}
{{end -}}
{{- end -}}
{{- end -}}
{{- end -}}

58
ceph-mon/values.yaml
View File

@ -28,7 +28,6 @@ images:
ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3'
ceph_mon: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
ceph_mon_check: 'docker.io/port/ceph-config-helper:v1.10.3'
fluentbit: docker.io/fluent/fluent-bit:0.12.14
dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
image_repo_sync: docker.io/docker:17.07.0
local_registry:
@ -71,13 +70,6 @@ pod:
limits:
memory: "50Mi"
cpu: "500m"
fluentbit:
requests:
memory: "5Mi"
cpu: "250m"
limits:
memory: "50Mi"
cpu: "500m"
jobs:
bootstrap:
limits:
@ -187,40 +179,6 @@ conf:
storage:
mon:
directory: /var/lib/openstack-helm/ceph/mon
fluentbit:
- service:
header: service
Flush: 30
Daemon: Off
Log_Level: info
Parsers_File: parsers.conf
- ceph_tail:
# NOTE(srwilkers): Despite being exposed, these values should not be
# modified, as the ceph-mon logs are always placed here
header: input
Name: tail
Tag: ceph-mon.*
Path: /var/log/ceph/*.log
Parser: syslog
DB: /var/log/ceph/ceph.db
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
Mem_Buf_Limit: 5MB
Refresh_Interval: 10s
parsers:
- syslog:
header: parser
Name: syslog
Format: regex
Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
Time_Key: time
Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
Time_Keep: On
Types: "pid:integer"
logging:
fluentd: false
dependencies:
dynamic:
@ -320,22 +278,6 @@ endpoints:
port:
mon:
default: 6789
fluentd:
namespace: null
name: fluentd
hosts:
default: fluentd-logging
host_fqdn_override:
default: null
path:
default: null
scheme:
default: http
port:
service:
default: 24224
metrics:
default: 24220
manifests:
configmap_bin: true

20
ceph-osd/templates/bin/osd/_fluentbit-sidecar.sh.tpl
View File

@ -1,20 +0,0 @@
#!/bin/sh
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf

4
ceph-osd/templates/configmap-bin.yaml
View File

@ -40,8 +40,4 @@ data:
{{ tuple "bin/osd/_stop.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
init-dirs.sh: |
{{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{ if .Values.logging.fluentd }}
fluentbit-sidecar.sh: |
{{ tuple "bin/osd/_fluentbit-sidecar.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{ end }}
{{- end }}

15
ceph-osd/templates/configmap-etc.yaml
View File

@ -40,15 +40,6 @@ limitations under the License.
{{- if empty .Values.conf.ceph.osd.public_network -}}
{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
{{- end -}}
{{- if not (has "fluentd_output" .Values.conf.fluentbit) -}}
{{- $fluentd_host := tuple "fluentd" "internal" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
{{- $fluentd_port := tuple "fluentd" "internal" "service" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $fluentd_output := dict "header" "output" "Name" "forward" "Match" "*" "Host" $fluentd_host "Port" $fluentd_port -}}
{{- $_ := set .Values "__fluentbit_config" ( list $fluentd_output) -}}
{{- $__fluentbit_config := append .Values.conf.fluentbit .Values.__fluentbit_config -}}
{{- $_ := set .Values.conf "fluentbit" $__fluentbit_config -}}
{{- end -}}
---
apiVersion: v1
kind: ConfigMap
@ -57,12 +48,6 @@ metadata:
data:
ceph.conf: |
{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
{{ if .Values.logging.fluentd }}
fluent-bit.conf: |
{{ include "ceph-osd.utils.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
parsers.conf: |
{{ include "ceph-osd.utils.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
{{ end }}
{{- end }}
{{- end }}
{{- if .Values.manifests.configmap_etc }}

59
ceph-osd/templates/daemonset-osd.yaml
View File

@ -75,6 +75,35 @@ spec:
- name: pod-run
mountPath: /run
readOnly: false
- name: ceph-log-ownership
{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
securityContext:
runAsUser: 0
env:
# NOTE(portdirect): These environment variables will be populated
# dynamicly at the point of render and added to all containers in the
# pod
# - name: JOURNAL_LOCATION
# value: /var/lib/openstack-helm/ceph/osd/journal-one
# - name: STORAGE_LOCATION
# value: /var/lib/openstack-helm/ceph/osd/data-one
# - name: JOURNAL_TYPE
# value: directory
# - name: STORAGE_TYPE
# value: directory
- name: CLUSTER
value: "ceph"
- name: CEPH_GET_ADMIN_KEY
value: "1"
command:
- chown
- -R
- ceph:root
- /var/log/ceph
volumeMounts:
- name: pod-var-log
mountPath: /var/log/ceph
readOnly: false
- name: osd-init
{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.osd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@ -218,30 +247,9 @@ spec:
- name: journal
mountPath: /var/lib/ceph/journal
readOnly: false
- name: varlog
- name: pod-var-log
mountPath: /var/log/ceph
{{ if .Values.logging.fluentd }}
- name: fluentbit-sidecar
{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
command:
- /tmp/fluentbit-sidecar.sh
volumeMounts:
- name: ceph-osd-bin
mountPath: /tmp/fluentbit-sidecar.sh
subPath: fluentbit-sidecar.sh
readOnly: true
- name: varlog
mountPath: /var/log/ceph
- name: ceph-osd-etc
mountPath: /fluent-bit/etc/fluent-bit.conf
subPath: fluent-bit.conf
readOnly: true
- name: ceph-osd-etc
mountPath: /fluent-bit/etc/parsers.conf
subPath: parsers.conf
readOnly: true
{{ end }}
readOnly: false
volumes:
- name: devices
hostPath:
@ -251,8 +259,9 @@ spec:
path: /run/lvm
- name: pod-var-lib-ceph
emptyDir: {}
- name: varlog
emptyDir: {}
- name: pod-var-log
hostPath:
path: /var/log/ceph
- name: pod-run
emptyDir:
medium: "Memory"

38
ceph-osd/templates/utils/_to_fluentbit_conf.tpl
View File

@ -1,38 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
# This function generates fluentbit configuration files with entries in the
# ceph-osd values.yaml. It results in a configuration section with the
# following format (for as many key/value pairs defined in values for a section):
# [HEADER]
# key value
# key value
# key value
# The configuration schema can be found here:
# http://fluentbit.io/documentation/0.12/configuration/schema.html
{{- define "ceph-osd.utils.to_fluentbit_conf" -}}
{{- range $values := . -}}
{{- range $section := . -}}
{{- $header := pick . "header" -}}
{{- $config := omit . "header" }}
[{{$header.header | upper }}]
{{range $key, $value := $config -}}
{{ $key | indent 4 }} {{ $value }}
{{end -}}
{{- end -}}
{{- end -}}
{{- end -}}

58
ceph-osd/values.yaml
View File

@ -22,7 +22,6 @@ images:
tags:
ceph_osd: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
fluentbit: docker.io/fluent/fluent-bit:0.12.14
dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
image_repo_sync: docker.io/docker:17.07.0
local_registry:
@ -56,13 +55,6 @@ pod:
limits:
memory: "1024Mi"
cpu: "1000m"
fluentbit:
requests:
memory: "5Mi"
cpu: "250m"
limits:
memory: "50Mi"
cpu: "500m"
jobs:
image_repo_sync:
requests:
@ -148,40 +140,6 @@ conf:
# journal:
# type: directory
# location: /var/lib/openstack-helm/ceph/osd/journal-three
fluentbit:
- service:
header: service
Flush: 30
Daemon: Off
Log_Level: info
Parsers_File: parsers.conf
- ceph_tail:
# NOTE(srwilkers): Despite being exposed, these values should not be
# modified, as the ceph-osd logs are always placed here
header: input
Name: tail
Tag: ceph-osd.*
Path: /var/log/ceph/ceph-osd.**.log
Parser: syslog
DB: /var/log/ceph/ceph-osd.db
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
Mem_Buf_Limit: 5MB
Refresh_Interval: 10s
parsers:
- syslog:
header: parser
Name: syslog
Format: regex
Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
Time_Key: time
Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
Time_Keep: On
Types: "pid:integer"
logging:
fluentd: false
dependencies:
dynamic:
@ -242,22 +200,6 @@ endpoints:
port:
mon:
default: 6789
fluentd:
namespace: null
name: fluentd
hosts:
default: fluentd-logging
host_fqdn_override:
default: null
path:
default: null
scheme:
default: http
port:
service:
default: 24224
metrics:
default: 24220
manifests:
configmap_bin: true

4
elasticsearch/values.yaml
View File

@ -106,8 +106,8 @@ pod:
default: kubernetes.io/hostname
replicas:
master: 3
data: 3
client: 2
data: 2
client: 3
lifecycle:
upgrades:
deployments:

4
fluent-logging/templates/utils/_to_fluentd_conf.tpl
View File

@ -62,6 +62,8 @@ section):
{{ $type | indent 4 }} {{ $innerValue }}
{{- else if contains "ENV" ($innerValue | quote) }}
{{ $innerKey | indent 4 }} {{ $innerValue | quote }}
{{- else if eq $innerKey "flush_interval" }}
{{ $innerKey | indent 4 }} {{ printf "%ss" $innerValue }}
{{- else }}
{{ $innerKey | indent 4 }} {{ $innerValue }}
{{- end }}
@ -75,6 +77,8 @@ section):
{{ $type | indent 2 }} {{ $value }}
{{- else if contains "ENV" ($value | quote) }}
{{ $key | indent 2 }} {{ $value | quote }}
{{- else if eq $key "flush_interval" }}
{{ $key | indent 2 }} {{ printf "%ss" $value }}
{{- else }}
{{ $key | indent 2 }} {{ $value }}
{{- end -}}

21
fluent-logging/values.yaml
View File

@ -301,9 +301,10 @@ conf:
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: journal
buffer_chunk_limit: 10M
buffer_queue_limit: 32
flush_interval: 20
buffer_chunk_limit: 2M
buffer_queue_limit: 8
flush_interval: "5"
flush_thread_count: 2
max_retry_wait: 300
disable_retry_limit: ""
num_threads: 8
@ -318,9 +319,10 @@ conf:
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: kernel
buffer_chunk_limit: 10M
buffer_queue_limit: 32
flush_interval: 20
buffer_chunk_limit: 2M
buffer_queue_limit: 8
flush_interval: "5"
flush_thread_count: 2
max_retry_wait: 300
disable_retry_limit: ""
num_threads: 8
@ -337,9 +339,10 @@ conf:
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
buffer_chunk_limit: 10M
buffer_queue_limit: 32
flush_interval: 20
buffer_chunk_limit: 2M
buffer_queue_limit: 8
flush_interval: "5"
flush_thread_count: 2
max_retry_wait: 300
disable_retry_limit: ""
num_threads: 8