Gates: Update fluent-logging/elasticsearch configurations

This updates the fluentd buffer output configurations to account
for the restraints of the jobs deploying fluentd. This also
renames the fluentd configuration key from td_agent to fluentd to
reflect the fact we're no longer deploying td-agent

This also updates the Elasticsearch default replicas and overrides
the replica counts in each Elasticsearch deployment to account for
resource constraints

Change-Id: I55dee410eced99c3e1645f7452e4306ad646e601
This commit is contained in:
Steve Wilkerson 2018-10-19 07:57:35 -05:00
parent 538d51e991
commit 4c29bafcbc
7 changed files with 10 additions and 10 deletions

View File

@ -106,7 +106,7 @@ pod:
default: kubernetes.io/hostname
replicas:
master: 3
data: 2
data: 3
client: 3
lifecycle:
upgrades:

View File

@ -34,7 +34,7 @@ function check_output_indexes_exist () {
Elasticsearch outputs should match. This means we can safely use the last item
in fluentd's configuration to get the Fluentd flush output interval
*/}}
{{- $fluentdConf := last .Values.conf.td_agent -}}
{{- $fluentdConf := last .Values.conf.fluentd -}}
{{- $fluentdElasticsearchConf := index $fluentdConf "elasticsearch" -}}
{{- $fluentdFlush := index $fluentdElasticsearchConf "flush_interval" -}}
fluentdFlush={{$fluentdFlush}}

View File

@ -25,7 +25,7 @@ type: Opaque
data:
fluent-bit.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.fluentbit | b64enc }}
parsers.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.parsers | b64enc }}
fluent.conf: {{ include "fluent_logging.utils.to_fluentd_conf" .Values.conf.td_agent | b64enc }}
fluent.conf: {{ include "fluent_logging.utils.to_fluentd_conf" .Values.conf.fluentd | b64enc }}
{{ range $template, $fields := .Values.conf.templates }}
{{ $template }}.json: {{ toJson $fields | b64enc }}
{{ end }}

View File

@ -202,7 +202,7 @@ conf:
Time_Key: time
Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
Time_Keep: On
td_agent:
fluentd:
- metrics_agent:
header: source
type: monitor_agent
@ -304,10 +304,8 @@ conf:
buffer_chunk_limit: 2M
buffer_queue_limit: 8
flush_interval: "5"
flush_thread_count: 2
max_retry_wait: 300
disable_retry_limit: ""
num_threads: 8
- kernel_elasticsearch:
header: match
type: elasticsearch
@ -322,10 +320,8 @@ conf:
buffer_chunk_limit: 2M
buffer_queue_limit: 8
flush_interval: "5"
flush_thread_count: 2
max_retry_wait: 300
disable_retry_limit: ""
num_threads: 8
# NOTE(srwilkers): This configuration entry should always be the last output
# defined, as it is used to determine the total flush cycle time for fluentbit
# and fluentd
@ -342,10 +338,8 @@ conf:
buffer_chunk_limit: 2M
buffer_queue_limit: 8
flush_interval: "5"
flush_thread_count: 2
max_retry_wait: 300
disable_retry_limit: ""
num_threads: 8
type_name: fluent
fluentd_exporter:
log:

View File

@ -21,6 +21,10 @@ make elasticsearch
#NOTE: Deploy command
tee /tmp/elasticsearch.yaml << EOF
pod:
replicas:
data: 1
master: 2
conf:
elasticsearch:
env:

View File

@ -27,6 +27,7 @@ monitoring:
pod:
replicas:
data: 1
master: 2
manifests:
network_policy: true
network_policy:

View File

@ -27,6 +27,7 @@ monitoring:
pod:
replicas:
data: 1
master: 2
conf:
elasticsearch:
snapshots: