diff --git a/monitor-helm/centos/monitor-helm.spec b/monitor-helm/centos/monitor-helm.spec index df222ac..a6b97ad 100644 --- a/monitor-helm/centos/monitor-helm.spec +++ b/monitor-helm/centos/monitor-helm.spec @@ -21,6 +21,7 @@ BuildArch: noarch Patch01: 0001-Add-Makefile-for-helm-charts.patch Patch02: 0002-kibana-workaround-checksum-for-configmap.yaml.patch Patch03: 0003-helm-chart-changes-for-stx-monitor.patch +Patch04: 0004-ipv6-helm-chart-changes.patch BuildRequires: helm @@ -32,6 +33,7 @@ Monitor Helm charts %patch01 -p1 %patch02 -p1 %patch03 -p1 +%patch04 -p1 %build # initialize helm and build the toolkit diff --git a/monitor-helm/files/0004-ipv6-helm-chart-changes.patch b/monitor-helm/files/0004-ipv6-helm-chart-changes.patch new file mode 100644 index 0000000..d555dfe --- /dev/null +++ b/monitor-helm/files/0004-ipv6-helm-chart-changes.patch @@ -0,0 +1,267 @@ +From 33fc3cb13670c8e76e7a93455865c81b9f13b0e0 Mon Sep 17 00:00:00 2001 +From: Kevin Smith +Date: Tue, 3 Sep 2019 10:43:50 -0400 +Subject: [PATCH 1/1] ipv6 helm chart changes + +--- + .../elasticsearch/templates/client-deployment.yaml | 2 +- + .../elasticsearch/templates/data-statefulset.yaml | 2 +- + .../templates/master-statefulset.yaml | 2 +- + stable/filebeat/templates/configmap.yaml | 47 +++++++++++++++++----- + stable/filebeat/templates/daemonset.yaml | 21 ++++++++++ + stable/metricbeat/templates/configmap.yaml | 47 +++++++++++++++++----- + stable/metricbeat/templates/daemonset.yaml | 21 ++++++++++ + 7 files changed, 119 insertions(+), 23 deletions(-) + +diff --git a/stable/elasticsearch/templates/client-deployment.yaml b/stable/elasticsearch/templates/client-deployment.yaml +index 047e3c8..5625ee0 100644 +--- a/stable/elasticsearch/templates/client-deployment.yaml ++++ b/stable/elasticsearch/templates/client-deployment.yaml +@@ -102,7 +102,7 @@ spec: + resourceFieldRef: + resource: limits.cpu + - name: ES_JAVA_OPTS +- value: "-Djava.net.preferIPv4Stack=true -Xms{{ .Values.client.heapSize }} -Xmx{{ .Values.client.heapSize }} {{ .Values.cluster.additionalJavaOpts }} {{ .Values.client.additionalJavaOpts }}" ++ value: "-Djava.net.preferIPv6Addresses=true -Xms{{ .Values.client.heapSize }} -Xmx{{ .Values.client.heapSize }} {{ .Values.cluster.additionalJavaOpts }} {{ .Values.client.additionalJavaOpts }}" + {{- range $key, $value := .Values.cluster.env }} + - name: {{ $key }} + value: {{ $value | quote }} +diff --git a/stable/elasticsearch/templates/data-statefulset.yaml b/stable/elasticsearch/templates/data-statefulset.yaml +index a407803..44b6be1 100644 +--- a/stable/elasticsearch/templates/data-statefulset.yaml ++++ b/stable/elasticsearch/templates/data-statefulset.yaml +@@ -120,7 +120,7 @@ spec: + resourceFieldRef: + resource: limits.cpu + - name: ES_JAVA_OPTS +- value: "-Djava.net.preferIPv4Stack=true -Xms{{ .Values.data.heapSize }} -Xmx{{ .Values.data.heapSize }} {{ .Values.cluster.additionalJavaOpts }} {{ .Values.data.additionalJavaOpts }}" ++ value: "-Djava.net.preferIPv6Addresses=true -Xms{{ .Values.data.heapSize }} -Xmx{{ .Values.data.heapSize }} {{ .Values.cluster.additionalJavaOpts }} {{ .Values.data.additionalJavaOpts }}" + {{- range $key, $value := .Values.cluster.env }} + - name: {{ $key }} + value: {{ $value | quote }} +diff --git a/stable/elasticsearch/templates/master-statefulset.yaml b/stable/elasticsearch/templates/master-statefulset.yaml +index 048535c..7c48fe3 100644 +--- a/stable/elasticsearch/templates/master-statefulset.yaml ++++ b/stable/elasticsearch/templates/master-statefulset.yaml +@@ -124,7 +124,7 @@ spec: + resourceFieldRef: + resource: limits.cpu + - name: ES_JAVA_OPTS +- value: "-Djava.net.preferIPv4Stack=true -Xms{{ .Values.master.heapSize }} -Xmx{{ .Values.master.heapSize }} {{ .Values.cluster.additionalJavaOpts }} {{ .Values.master.additionalJavaOpts }}" ++ value: "-Djava.net.preferIPv6Addresses=true -Xms{{ .Values.master.heapSize }} -Xmx{{ .Values.master.heapSize }} {{ .Values.cluster.additionalJavaOpts }} {{ .Values.master.additionalJavaOpts }}" + {{- range $key, $value := .Values.cluster.env }} + - name: {{ $key }} + value: {{ $value | quote }} +diff --git a/stable/filebeat/templates/configmap.yaml b/stable/filebeat/templates/configmap.yaml +index c895965..9433176 100644 +--- a/stable/filebeat/templates/configmap.yaml ++++ b/stable/filebeat/templates/configmap.yaml +@@ -10,8 +10,41 @@ metadata: + data: + setup-script.sh: |- + #!/bin/bash ++ # ++ # This is best effort to load the template into elasticsearch ++ # if beats are going to elasticsearch via logstash. ++ # + BEAT='filebeat' + BEAT_VER=$($BEAT version | awk '{print $3}') ++ ++ ADDR=$OUTPUT_ELASTICSEARCH_HOSTS ++ echo $ADDR ++ ++ # Check if this is a map of entries. If so, we'll only export the template ++ # to the first address ++ if [[ ${ADDR:0:1} == [ ]]; then ++ # Remove the square brackets ++ ADDR=${ADDR:1:-1} ++ # Take the first entry in case there are more than one ++ ADDR=${ADDR%%,*} ++ fi ++ echo $ADDR ++ ++ # Check if user has formatted with http:// on front, if not we need to add ++ HTTP='http://' ++ if [[ ${ADDR} == http* ]]; then ++ HTTP='' ++ fi ++ echo $HTTP ++ ++ # Check for ip address containing special characters where -g must be used in curl command ++ # IPV6 Addresses should come in with square braces around the address. ++ CURL_G='' ++ if [[ ${ADDR} == *[* ]]; then ++ CURL_G=' -g ' ++ fi ++ echo $CURL_G ++ + # check if logstash output is enabled, via crude parsing of the .yml file. + sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true + if [[ $? -eq 0 ]]; then +@@ -20,15 +53,9 @@ data: + # remove the lifecycle section of the yaml, as elasticsearch will choke + # on it as oss version does not support ilm. + sed -i '/lifecycle/,+3d' /tmp/beat.template.json +- curl -XPUT -H "Content-Type: application/json" http://{$STX_ELASTICSEARCH_CLIENT_SERVICE_HOST}:{$STX_ELASTICSEARCH_CLIENT_SERVICE_PORT_HTTP}/_template/{$BEAT}-{$BEAT_VER} -d@/tmp/beat.template.json +- # prime the index as a write index +- curl -XPUT -H "Content-Type: application/json" -d "{ +- \"aliases\": { +- \"{BEAT}-{$BEAT_VER}\": { +- \"is_write_index\": \"true\" +- } +- } +- }" http://{$STX_ELASTICSEARCH_CLIENT_SERVICE_HOST}:{$STX_ELASTICSEARCH_CLIENT_SERVICE_PORT_HTTP}/{$BEAT}-{$BEAT_VER}-000001 ++ curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json + else +- echo "logstash not configured, not exporting template, should be done for us." ++ echo "logstash not configured, not exporting template, should be done for us." + fi ++ # Above is non-fatal if there is a problem. Always pass. ++ exit 0 +diff --git a/stable/filebeat/templates/daemonset.yaml b/stable/filebeat/templates/daemonset.yaml +index 00a5519..b8536fb 100644 +--- a/stable/filebeat/templates/daemonset.yaml ++++ b/stable/filebeat/templates/daemonset.yaml +@@ -53,6 +53,17 @@ spec: + mountPath: /usr/share/filebeat/filebeat.yml + readOnly: true + subPath: filebeat.yml ++ env: ++ ## make output.elasticsearch parms visible ++ {{- range $key, $value := .Values.config}} ++ {{- $configname := $key | upper | replace "." "_" -}} ++ {{ if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} + {{- if .Values.indexTemplateLoad }} + - name: "load-es-template" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +@@ -95,6 +106,16 @@ spec: + valueFrom: + fieldRef: + fieldPath: spec.nodeName ++ ## make output.elasticsearch parms visible ++ {{- range $key, $value := .Values.config}} ++ {{- $configname := $key | upper | replace "." "_" -}} ++ {{ if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} + {{- if .Values.extraVars }} + {{ toYaml .Values.extraVars | indent 8 }} + {{- end }} +diff --git a/stable/metricbeat/templates/configmap.yaml b/stable/metricbeat/templates/configmap.yaml +index 21890b4..f990f0c 100644 +--- a/stable/metricbeat/templates/configmap.yaml ++++ b/stable/metricbeat/templates/configmap.yaml +@@ -10,8 +10,41 @@ metadata: + data: + setup-script.sh: |- + #!/bin/bash ++ # ++ # This is best effort to load the template into elasticsearch ++ # if beats are going to elasticsearch via logstash. ++ # + BEAT='metricbeat' + BEAT_VER=$($BEAT version | awk '{print $3}') ++ ++ ADDR=$OUTPUT_ELASTICSEARCH_HOSTS ++ echo $ADDR ++ ++ # Check if this is a map of entries. If so, we'll only export the template ++ # to the first address ++ if [[ ${ADDR:0:1} == [ ]]; then ++ # Remove the square brackets ++ ADDR=${ADDR:1:-1} ++ # Take the first entry in case there are more than one ++ ADDR=${ADDR%%,*} ++ fi ++ echo $ADDR ++ ++ # Check if user has formatted with http:// on front, if not we need to add ++ HTTP='http://' ++ if [[ ${ADDR} == http* ]]; then ++ HTTP='' ++ fi ++ echo $HTTP ++ ++ # Check for ip address containing special characters where -g must be used in curl command ++ # IPV6 Addresses should come in with square braces around the address. ++ CURL_G='' ++ if [[ ${ADDR} == *[* ]]; then ++ CURL_G=' -g ' ++ fi ++ echo $CURL_G ++ + # check if logstash output is enabled, via crude parsing of the .yml file. + sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true + if [[ $? -eq 0 ]]; then +@@ -20,15 +53,9 @@ data: + # remove the lifecycle section of the yaml, as elasticsearch will choke + # on it as oss version does not support ilm. + sed -i '/lifecycle/,+3d' /tmp/beat.template.json +- curl -XPUT -H "Content-Type: application/json" http://{$STX_ELASTICSEARCH_CLIENT_SERVICE_HOST}:{$STX_ELASTICSEARCH_CLIENT_SERVICE_PORT_HTTP}/_template/{$BEAT}-{$BEAT_VER} -d@/tmp/beat.template.json +- # prime the index as a write index +- curl -XPUT -H "Content-Type: application/json" -d "{ +- \"aliases\": { +- \"{BEAT}-{$BEAT_VER}\": { +- \"is_write_index\": \"true\" +- } +- } +- }" http://{$STX_ELASTICSEARCH_CLIENT_SERVICE_HOST}:{$STX_ELASTICSEARCH_CLIENT_SERVICE_PORT_HTTP}/{$BEAT}-{$BEAT_VER}-000001 ++ curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json + else +- echo "logstash not configured, not exporting template, should be done for us." ++ echo "logstash not configured, not exporting template, should be done for us." + fi ++ # Above is non-fatal if there is a problem. Always pass. ++ exit 0 +diff --git a/stable/metricbeat/templates/daemonset.yaml b/stable/metricbeat/templates/daemonset.yaml +index e3d7437..1ae3a34 100644 +--- a/stable/metricbeat/templates/daemonset.yaml ++++ b/stable/metricbeat/templates/daemonset.yaml +@@ -49,6 +49,17 @@ spec: + readOnly: true + subPath: metricbeat.yml + env: ++ ## make output.elasticsearch parms visible ++ {{- range $key, $value := .Values.daemonset.config}} ++ {{- $configname := $key | upper | replace "." "_" -}} ++ {{ if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +@@ -73,6 +84,16 @@ spec: + valueFrom: + fieldRef: + fieldPath: spec.nodeName ++ ## make output.elasticsearch parms visible ++ {{- range $key, $value := .Values.daemonset.config}} ++ {{- $configname := $key | upper | replace "." "_" -}} ++ {{ if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} + {{- if .Values.extraEnv }} + {{ toYaml .Values.extraEnv | indent 8 }} + {{- end }} +-- +1.8.3.1 + diff --git a/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml b/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml index fbcbe05..97f1ff5 100644 --- a/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml +++ b/stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml @@ -370,6 +370,7 @@ data: files: kibana.yml: elasticsearch.hosts: http://stx-elasticsearch-client:9200 + server.host: "::" nodeSelector: elastic-controller: "enabled" source: @@ -425,14 +426,14 @@ data: elasticsearch { hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}${ELASTICSEARCH_PATH}"] manage_template => false - index => "collectd" + index => "collectd-%{+YYYY.MM.dd}" } } if [type] == "beats" { elasticsearch { hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}${ELASTICSEARCH_PATH}"] manage_template => false - index => "%{[@metadata][beat]}-%{[@metadata][version]}" + index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" } } } @@ -442,6 +443,7 @@ data: udp { #Note default port 25826 in use by influxdb. #port => 25826 + host => "::" port => 31005 buffer_size => 1452 codec => collectd { }